2024-11-20 10:23:51,971 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 10:23:51,982 main DEBUG Took 0.009158 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 10:23:51,983 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 10:23:51,983 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 10:23:51,984 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 10:23:51,985 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:23:51,993 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 10:23:52,006 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:23:52,008 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:23:52,009 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:23:52,010 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:23:52,010 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:23:52,011 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:23:52,012 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:23:52,013 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:23:52,014 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:23:52,014 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:23:52,015 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:23:52,016 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:23:52,016 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:23:52,017 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:23:52,017 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:23:52,018 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:23:52,018 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:23:52,019 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:23:52,020 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:23:52,020 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:23:52,021 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:23:52,021 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:23:52,022 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:23:52,022 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:23:52,023 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:23:52,023 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 10:23:52,025 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:23:52,026 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 10:23:52,029 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 10:23:52,029 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 10:23:52,031 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 10:23:52,031 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 10:23:52,041 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 10:23:52,044 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 10:23:52,047 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 10:23:52,047 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 10:23:52,048 main DEBUG createAppenders(={Console}) 2024-11-20 10:23:52,049 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-20 10:23:52,049 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 10:23:52,049 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-20 10:23:52,050 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 10:23:52,050 main DEBUG OutputStream closed 2024-11-20 10:23:52,051 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 10:23:52,051 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 10:23:52,051 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-20 10:23:52,131 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 10:23:52,134 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 10:23:52,135 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 10:23:52,136 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 10:23:52,136 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 10:23:52,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 10:23:52,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 10:23:52,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 10:23:52,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 10:23:52,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 10:23:52,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 10:23:52,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 10:23:52,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 10:23:52,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 10:23:52,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 10:23:52,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 10:23:52,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 10:23:52,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 10:23:52,143 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 10:23:52,143 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-20 10:23:52,143 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 10:23:52,144 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-20T10:23:52,366 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605 2024-11-20 10:23:52,369 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 10:23:52,369 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T10:23:52,377 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-20T10:23:52,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T10:23:52,411 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/cluster_df478fc5-be47-d3ac-f609-45ae39353117, deleteOnExit=true 2024-11-20T10:23:52,411 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T10:23:52,412 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/test.cache.data in system properties and HBase conf 2024-11-20T10:23:52,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T10:23:52,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/hadoop.log.dir in system properties and HBase conf 2024-11-20T10:23:52,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T10:23:52,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T10:23:52,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T10:23:52,499 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T10:23:52,606 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T10:23:52,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T10:23:52,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T10:23:52,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T10:23:52,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T10:23:52,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T10:23:52,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T10:23:52,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T10:23:52,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T10:23:52,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T10:23:52,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/nfs.dump.dir in system properties and HBase conf 2024-11-20T10:23:52,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/java.io.tmpdir in system properties and HBase conf 2024-11-20T10:23:52,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T10:23:52,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T10:23:52,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T10:23:53,749 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T10:23:53,862 INFO [Time-limited test {}] log.Log(170): Logging initialized @2571ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T10:23:53,967 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T10:23:54,068 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T10:23:54,104 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T10:23:54,105 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T10:23:54,107 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T10:23:54,125 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T10:23:54,130 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/hadoop.log.dir/,AVAILABLE} 2024-11-20T10:23:54,131 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T10:23:54,402 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/java.io.tmpdir/jetty-localhost-39013-hadoop-hdfs-3_4_1-tests_jar-_-any-1654326545774414280/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T10:23:54,411 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:39013} 2024-11-20T10:23:54,411 INFO [Time-limited test {}] server.Server(415): Started @3121ms 2024-11-20T10:23:55,086 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T10:23:55,098 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T10:23:55,099 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T10:23:55,099 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T10:23:55,100 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T10:23:55,103 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6ff64ba0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/hadoop.log.dir/,AVAILABLE} 2024-11-20T10:23:55,103 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4828e07b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T10:23:55,213 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@51084f0f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/java.io.tmpdir/jetty-localhost-43027-hadoop-hdfs-3_4_1-tests_jar-_-any-7779867426265884584/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T10:23:55,214 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@370621d9{HTTP/1.1, (http/1.1)}{localhost:43027} 2024-11-20T10:23:55,214 INFO [Time-limited test {}] server.Server(415): Started @3924ms 2024-11-20T10:23:55,267 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T10:23:56,404 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/cluster_df478fc5-be47-d3ac-f609-45ae39353117/data/data1/current/BP-606364108-172.17.0.2-1732098233200/current, will proceed with Du for space computation calculation, 2024-11-20T10:23:56,406 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/cluster_df478fc5-be47-d3ac-f609-45ae39353117/data/data2/current/BP-606364108-172.17.0.2-1732098233200/current, will proceed with Du for space computation calculation, 2024-11-20T10:23:56,444 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T10:23:56,502 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfa9879640fe9918 with lease ID 0xc0d8f173fc0f1b3e: Processing first storage report for DS-420d2dd3-5eb8-4b12-9929-9530a85142e0 from datanode DatanodeRegistration(127.0.0.1:37423, datanodeUuid=ae59ac02-3972-44ab-8c09-49370f1bb9eb, infoPort=38461, infoSecurePort=0, ipcPort=36823, storageInfo=lv=-57;cid=testClusterID;nsid=687986718;c=1732098233200) 2024-11-20T10:23:56,503 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfa9879640fe9918 with lease ID 0xc0d8f173fc0f1b3e: from storage DS-420d2dd3-5eb8-4b12-9929-9530a85142e0 node DatanodeRegistration(127.0.0.1:37423, datanodeUuid=ae59ac02-3972-44ab-8c09-49370f1bb9eb, infoPort=38461, infoSecurePort=0, ipcPort=36823, storageInfo=lv=-57;cid=testClusterID;nsid=687986718;c=1732098233200), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T10:23:56,504 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfa9879640fe9918 with lease ID 0xc0d8f173fc0f1b3e: Processing first storage report for DS-b87b3d80-df96-41eb-97b5-761f9fc9560e from datanode DatanodeRegistration(127.0.0.1:37423, datanodeUuid=ae59ac02-3972-44ab-8c09-49370f1bb9eb, infoPort=38461, infoSecurePort=0, ipcPort=36823, storageInfo=lv=-57;cid=testClusterID;nsid=687986718;c=1732098233200) 2024-11-20T10:23:56,504 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfa9879640fe9918 with lease ID 0xc0d8f173fc0f1b3e: from storage DS-b87b3d80-df96-41eb-97b5-761f9fc9560e node DatanodeRegistration(127.0.0.1:37423, datanodeUuid=ae59ac02-3972-44ab-8c09-49370f1bb9eb, infoPort=38461, infoSecurePort=0, ipcPort=36823, storageInfo=lv=-57;cid=testClusterID;nsid=687986718;c=1732098233200), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T10:23:56,538 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605 2024-11-20T10:23:56,607 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/cluster_df478fc5-be47-d3ac-f609-45ae39353117/zookeeper_0, clientPort=55353, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/cluster_df478fc5-be47-d3ac-f609-45ae39353117/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/cluster_df478fc5-be47-d3ac-f609-45ae39353117/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T10:23:56,618 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55353 2024-11-20T10:23:56,628 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:23:56,632 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:23:56,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741825_1001 (size=7) 2024-11-20T10:23:57,239 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31 with version=8 2024-11-20T10:23:57,240 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/hbase-staging 2024-11-20T10:23:57,347 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T10:23:57,643 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8c533afd331c:0 server-side Connection retries=45 2024-11-20T10:23:57,652 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T10:23:57,653 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T10:23:57,658 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T10:23:57,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T10:23:57,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T10:23:57,833 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T10:23:57,891 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T10:23:57,900 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T10:23:57,904 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T10:23:57,929 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 33068 (auto-detected) 2024-11-20T10:23:57,930 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T10:23:57,949 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33867 2024-11-20T10:23:57,976 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33867 connecting to ZooKeeper ensemble=127.0.0.1:55353 2024-11-20T10:23:58,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:338670x0, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T10:23:58,102 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33867-0x10157c59c580000 connected 2024-11-20T10:23:58,564 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:23:58,567 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:23:58,577 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T10:23:58,581 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31, hbase.cluster.distributed=false 2024-11-20T10:23:58,602 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T10:23:58,607 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33867 2024-11-20T10:23:58,607 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33867 2024-11-20T10:23:58,608 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33867 2024-11-20T10:23:58,610 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33867 2024-11-20T10:23:58,612 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33867 2024-11-20T10:23:58,712 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8c533afd331c:0 server-side Connection retries=45 2024-11-20T10:23:58,714 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T10:23:58,714 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T10:23:58,714 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T10:23:58,714 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T10:23:58,715 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T10:23:58,717 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T10:23:58,720 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T10:23:58,721 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35833 2024-11-20T10:23:58,723 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35833 connecting to ZooKeeper ensemble=127.0.0.1:55353 2024-11-20T10:23:58,724 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:23:58,728 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:23:58,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:358330x0, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T10:23:58,750 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:358330x0, quorum=127.0.0.1:55353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T10:23:58,750 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35833-0x10157c59c580001 connected 2024-11-20T10:23:58,754 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T10:23:58,770 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T10:23:58,773 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T10:23:58,780 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T10:23:58,784 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35833 2024-11-20T10:23:58,784 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35833 2024-11-20T10:23:58,785 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35833 2024-11-20T10:23:58,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35833 2024-11-20T10:23:58,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35833 2024-11-20T10:23:58,807 DEBUG [M:0;8c533afd331c:33867 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8c533afd331c:33867 2024-11-20T10:23:58,808 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8c533afd331c,33867,1732098237476 2024-11-20T10:23:58,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T10:23:58,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T10:23:58,825 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8c533afd331c,33867,1732098237476 2024-11-20T10:23:58,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T10:23:58,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:23:58,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:23:58,855 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T10:23:58,856 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8c533afd331c,33867,1732098237476 from backup master directory 2024-11-20T10:23:58,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8c533afd331c,33867,1732098237476 2024-11-20T10:23:58,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T10:23:58,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T10:23:58,865 WARN [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T10:23:58,865 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8c533afd331c,33867,1732098237476 2024-11-20T10:23:58,867 INFO [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T10:23:58,869 INFO [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T10:23:58,932 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/hbase.id] with ID: 19b99c7a-1668-4a27-a15b-962d0dae8a19 2024-11-20T10:23:58,932 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/.tmp/hbase.id 2024-11-20T10:23:58,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741826_1002 (size=42) 2024-11-20T10:23:59,345 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/.tmp/hbase.id]:[hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/hbase.id] 2024-11-20T10:23:59,393 INFO [master/8c533afd331c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:23:59,398 INFO [master/8c533afd331c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T10:23:59,415 INFO [master/8c533afd331c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-20T10:23:59,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:23:59,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:23:59,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741827_1003 (size=196) 2024-11-20T10:23:59,903 INFO [master/8c533afd331c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T10:23:59,906 INFO [master/8c533afd331c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T10:23:59,923 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:23:59,928 INFO [master/8c533afd331c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:23:59,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741828_1004 (size=1189) 2024-11-20T10:24:00,382 INFO [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store 2024-11-20T10:24:00,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741829_1005 (size=34) 2024-11-20T10:24:00,803 INFO [master/8c533afd331c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T10:24:00,807 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:24:00,808 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T10:24:00,809 INFO [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:24:00,809 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:24:00,811 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T10:24:00,811 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:24:00,811 INFO [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:24:00,813 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732098240808Disabling compacts and flushes for region at 1732098240808Disabling writes for close at 1732098240811 (+3 ms)Writing region close event to WAL at 1732098240811Closed at 1732098240811 2024-11-20T10:24:00,815 WARN [master/8c533afd331c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/.initializing 2024-11-20T10:24:00,815 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/WALs/8c533afd331c,33867,1732098237476 2024-11-20T10:24:00,824 INFO [master/8c533afd331c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:24:00,839 INFO [master/8c533afd331c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8c533afd331c%2C33867%2C1732098237476, suffix=, logDir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/WALs/8c533afd331c,33867,1732098237476, archiveDir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/oldWALs, maxLogs=10 2024-11-20T10:24:00,870 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/WALs/8c533afd331c,33867,1732098237476/8c533afd331c%2C33867%2C1732098237476.1732098240845, exclude list is [], retry=0 2024-11-20T10:24:00,889 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37423,DS-420d2dd3-5eb8-4b12-9929-9530a85142e0,DISK] 2024-11-20T10:24:00,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T10:24:00,925 INFO [master/8c533afd331c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/WALs/8c533afd331c,33867,1732098237476/8c533afd331c%2C33867%2C1732098237476.1732098240845 2024-11-20T10:24:00,927 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38461:38461)] 2024-11-20T10:24:00,927 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:24:00,928 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:24:00,931 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:24:00,933 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:24:00,969 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:24:00,993 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T10:24:00,996 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:00,999 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:24:00,999 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:24:01,003 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T10:24:01,004 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:01,005 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:24:01,006 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:24:01,009 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T10:24:01,009 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:01,011 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:24:01,011 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:24:01,015 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T10:24:01,015 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:01,016 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:24:01,017 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:24:01,021 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:24:01,023 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:24:01,030 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:24:01,031 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:24:01,035 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T10:24:01,040 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:24:01,046 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:24:01,048 INFO [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71766594, jitterRate=0.06940558552742004}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T10:24:01,056 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732098240945Initializing all the Stores at 1732098240947 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098240948 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098240949 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098240949Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098240949Cleaning up temporary data from old regions at 1732098241031 (+82 ms)Region opened successfully at 1732098241056 (+25 ms) 2024-11-20T10:24:01,058 INFO [master/8c533afd331c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T10:24:01,098 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d774ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8c533afd331c/172.17.0.2:0 2024-11-20T10:24:01,140 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T10:24:01,154 INFO [master/8c533afd331c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T10:24:01,155 INFO [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T10:24:01,158 INFO [master/8c533afd331c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T10:24:01,159 INFO [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T10:24:01,164 INFO [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T10:24:01,164 INFO [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T10:24:01,190 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T10:24:01,199 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T10:24:01,296 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T10:24:01,299 INFO [master/8c533afd331c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T10:24:01,302 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T10:24:01,315 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T10:24:01,318 INFO [master/8c533afd331c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T10:24:01,322 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T10:24:01,327 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T10:24:01,329 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T10:24:01,338 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T10:24:01,360 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T10:24:01,369 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T10:24:01,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T10:24:01,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:24:01,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T10:24:01,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:24:01,384 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8c533afd331c,33867,1732098237476, sessionid=0x10157c59c580000, setting cluster-up flag (Was=false) 2024-11-20T10:24:01,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:24:01,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:24:01,443 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T10:24:01,445 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8c533afd331c,33867,1732098237476 2024-11-20T10:24:01,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:24:01,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:24:01,495 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T10:24:01,497 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8c533afd331c,33867,1732098237476 2024-11-20T10:24:01,504 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T10:24:01,595 INFO [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(746): ClusterId : 19b99c7a-1668-4a27-a15b-962d0dae8a19 2024-11-20T10:24:01,598 DEBUG [RS:0;8c533afd331c:35833 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T10:24:01,601 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T10:24:01,613 INFO [master/8c533afd331c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T10:24:01,614 DEBUG [RS:0;8c533afd331c:35833 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T10:24:01,614 DEBUG [RS:0;8c533afd331c:35833 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T10:24:01,621 INFO [master/8c533afd331c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T10:24:01,624 DEBUG [RS:0;8c533afd331c:35833 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T10:24:01,624 DEBUG [RS:0;8c533afd331c:35833 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4906a00e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8c533afd331c/172.17.0.2:0 2024-11-20T10:24:01,628 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8c533afd331c,33867,1732098237476 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T10:24:01,637 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8c533afd331c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T10:24:01,637 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8c533afd331c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T10:24:01,638 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8c533afd331c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T10:24:01,638 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8c533afd331c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T10:24:01,638 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8c533afd331c:0, corePoolSize=10, maxPoolSize=10 2024-11-20T10:24:01,638 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:24:01,638 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8c533afd331c:0, corePoolSize=2, maxPoolSize=2 2024-11-20T10:24:01,639 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:24:01,646 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T10:24:01,646 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8c533afd331c:35833 2024-11-20T10:24:01,646 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T10:24:01,648 INFO [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732098271648 2024-11-20T10:24:01,650 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T10:24:01,650 INFO [RS:0;8c533afd331c:35833 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T10:24:01,650 INFO [RS:0;8c533afd331c:35833 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T10:24:01,651 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T10:24:01,651 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T10:24:01,653 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:01,654 INFO [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(2659): reportForDuty to master=8c533afd331c,33867,1732098237476 with port=35833, startcode=1732098238678 2024-11-20T10:24:01,653 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T10:24:01,656 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T10:24:01,656 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T10:24:01,657 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T10:24:01,658 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T10:24:01,659 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:01,672 DEBUG [RS:0;8c533afd331c:35833 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T10:24:01,677 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T10:24:01,679 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T10:24:01,679 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T10:24:01,684 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T10:24:01,684 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T10:24:01,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741831_1007 (size=1321) 2024-11-20T10:24:01,688 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8c533afd331c:0:becomeActiveMaster-HFileCleaner.large.0-1732098241686,5,FailOnTimeoutGroup] 2024-11-20T10:24:01,692 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8c533afd331c:0:becomeActiveMaster-HFileCleaner.small.0-1732098241688,5,FailOnTimeoutGroup] 2024-11-20T10:24:01,692 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:01,693 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T10:24:01,694 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:01,694 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:01,720 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32991, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T10:24:01,726 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33867 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8c533afd331c,35833,1732098238678 2024-11-20T10:24:01,729 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33867 {}] master.ServerManager(517): Registering regionserver=8c533afd331c,35833,1732098238678 2024-11-20T10:24:01,745 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31 2024-11-20T10:24:01,745 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45087 2024-11-20T10:24:01,745 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T10:24:01,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T10:24:01,760 DEBUG [RS:0;8c533afd331c:35833 {}] zookeeper.ZKUtil(111): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8c533afd331c,35833,1732098238678 2024-11-20T10:24:01,761 WARN [RS:0;8c533afd331c:35833 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T10:24:01,761 INFO [RS:0;8c533afd331c:35833 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:24:01,761 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/WALs/8c533afd331c,35833,1732098238678 2024-11-20T10:24:01,763 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8c533afd331c,35833,1732098238678] 2024-11-20T10:24:01,793 INFO [RS:0;8c533afd331c:35833 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T10:24:01,815 INFO [RS:0;8c533afd331c:35833 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T10:24:01,820 INFO [RS:0;8c533afd331c:35833 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T10:24:01,820 INFO [RS:0;8c533afd331c:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:01,821 INFO [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T10:24:01,826 INFO [RS:0;8c533afd331c:35833 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T10:24:01,827 INFO [RS:0;8c533afd331c:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:01,827 DEBUG [RS:0;8c533afd331c:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:24:01,827 DEBUG [RS:0;8c533afd331c:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:24:01,828 DEBUG [RS:0;8c533afd331c:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:24:01,828 DEBUG [RS:0;8c533afd331c:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:24:01,828 DEBUG [RS:0;8c533afd331c:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:24:01,828 DEBUG [RS:0;8c533afd331c:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0, corePoolSize=2, maxPoolSize=2 2024-11-20T10:24:01,828 DEBUG [RS:0;8c533afd331c:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:24:01,828 DEBUG [RS:0;8c533afd331c:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:24:01,828 DEBUG [RS:0;8c533afd331c:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:24:01,829 DEBUG [RS:0;8c533afd331c:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:24:01,829 DEBUG [RS:0;8c533afd331c:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:24:01,829 DEBUG [RS:0;8c533afd331c:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:24:01,829 DEBUG [RS:0;8c533afd331c:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8c533afd331c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T10:24:01,829 DEBUG [RS:0;8c533afd331c:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T10:24:01,832 INFO [RS:0;8c533afd331c:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:01,832 INFO [RS:0;8c533afd331c:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:01,833 INFO [RS:0;8c533afd331c:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:01,833 INFO [RS:0;8c533afd331c:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:01,833 INFO [RS:0;8c533afd331c:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:01,833 INFO [RS:0;8c533afd331c:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,35833,1732098238678-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T10:24:01,851 INFO [RS:0;8c533afd331c:35833 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T10:24:01,852 INFO [RS:0;8c533afd331c:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,35833,1732098238678-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:01,852 INFO [RS:0;8c533afd331c:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:01,853 INFO [RS:0;8c533afd331c:35833 {}] regionserver.Replication(171): 8c533afd331c,35833,1732098238678 started 2024-11-20T10:24:01,874 INFO [RS:0;8c533afd331c:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:01,875 INFO [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1482): Serving as 8c533afd331c,35833,1732098238678, RpcServer on 8c533afd331c/172.17.0.2:35833, sessionid=0x10157c59c580001 2024-11-20T10:24:01,875 DEBUG [RS:0;8c533afd331c:35833 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T10:24:01,876 DEBUG [RS:0;8c533afd331c:35833 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8c533afd331c,35833,1732098238678 2024-11-20T10:24:01,876 DEBUG [RS:0;8c533afd331c:35833 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8c533afd331c,35833,1732098238678' 2024-11-20T10:24:01,876 DEBUG [RS:0;8c533afd331c:35833 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T10:24:01,877 DEBUG [RS:0;8c533afd331c:35833 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T10:24:01,877 DEBUG [RS:0;8c533afd331c:35833 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T10:24:01,877 DEBUG [RS:0;8c533afd331c:35833 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T10:24:01,878 DEBUG [RS:0;8c533afd331c:35833 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8c533afd331c,35833,1732098238678 2024-11-20T10:24:01,878 DEBUG [RS:0;8c533afd331c:35833 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8c533afd331c,35833,1732098238678' 2024-11-20T10:24:01,878 DEBUG [RS:0;8c533afd331c:35833 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T10:24:01,878 DEBUG [RS:0;8c533afd331c:35833 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T10:24:01,879 DEBUG [RS:0;8c533afd331c:35833 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T10:24:01,879 INFO [RS:0;8c533afd331c:35833 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T10:24:01,879 INFO [RS:0;8c533afd331c:35833 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T10:24:01,983 INFO [RS:0;8c533afd331c:35833 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:24:01,987 INFO [RS:0;8c533afd331c:35833 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8c533afd331c%2C35833%2C1732098238678, suffix=, logDir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/WALs/8c533afd331c,35833,1732098238678, archiveDir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/oldWALs, maxLogs=32 2024-11-20T10:24:02,003 DEBUG [RS:0;8c533afd331c:35833 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/WALs/8c533afd331c,35833,1732098238678/8c533afd331c%2C35833%2C1732098238678.1732098241991, exclude list is [], retry=0 2024-11-20T10:24:02,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37423,DS-420d2dd3-5eb8-4b12-9929-9530a85142e0,DISK] 2024-11-20T10:24:02,011 INFO [RS:0;8c533afd331c:35833 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/WALs/8c533afd331c,35833,1732098238678/8c533afd331c%2C35833%2C1732098238678.1732098241991 2024-11-20T10:24:02,011 DEBUG [RS:0;8c533afd331c:35833 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38461:38461)] 2024-11-20T10:24:02,090 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T10:24:02,091 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31 2024-11-20T10:24:02,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741833_1009 (size=32) 2024-11-20T10:24:02,511 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:24:02,514 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T10:24:02,517 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T10:24:02,517 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:02,518 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:24:02,518 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T10:24:02,520 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T10:24:02,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:02,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:24:02,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T10:24:02,524 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T10:24:02,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:02,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:24:02,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T10:24:02,529 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T10:24:02,529 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:02,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:24:02,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T10:24:02,532 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740 2024-11-20T10:24:02,532 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740 2024-11-20T10:24:02,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T10:24:02,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T10:24:02,536 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T10:24:02,538 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T10:24:02,542 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:24:02,543 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67587801, jitterRate=0.007136717438697815}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T10:24:02,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732098242511Initializing all the Stores at 1732098242514 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098242514Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098242514Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098242514Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098242514Cleaning up temporary data from old regions at 1732098242535 (+21 ms)Region opened successfully at 1732098242546 (+11 ms) 2024-11-20T10:24:02,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T10:24:02,546 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T10:24:02,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T10:24:02,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T10:24:02,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T10:24:02,548 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T10:24:02,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732098242546Disabling compacts and flushes for region at 1732098242546Disabling writes for close at 1732098242546Writing region close event to WAL at 1732098242547 (+1 ms)Closed at 1732098242548 (+1 ms) 2024-11-20T10:24:02,551 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T10:24:02,551 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T10:24:02,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T10:24:02,563 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T10:24:02,566 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T10:24:02,722 DEBUG [8c533afd331c:33867 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-20T10:24:02,736 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:24:02,742 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8c533afd331c,35833,1732098238678, state=OPENING 2024-11-20T10:24:02,811 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T10:24:02,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:24:02,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:24:02,824 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:24:02,824 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:24:02,826 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T10:24:02,828 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:24:03,007 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T10:24:03,010 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39203, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T10:24:03,020 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T10:24:03,020 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:24:03,021 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T10:24:03,025 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8c533afd331c%2C35833%2C1732098238678.meta, suffix=.meta, logDir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/WALs/8c533afd331c,35833,1732098238678, archiveDir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/oldWALs, maxLogs=32 2024-11-20T10:24:03,040 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/WALs/8c533afd331c,35833,1732098238678/8c533afd331c%2C35833%2C1732098238678.meta.1732098243027.meta, exclude list is [], retry=0 2024-11-20T10:24:03,044 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37423,DS-420d2dd3-5eb8-4b12-9929-9530a85142e0,DISK] 2024-11-20T10:24:03,047 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/WALs/8c533afd331c,35833,1732098238678/8c533afd331c%2C35833%2C1732098238678.meta.1732098243027.meta 2024-11-20T10:24:03,047 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38461:38461)] 2024-11-20T10:24:03,048 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:24:03,050 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T10:24:03,053 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T10:24:03,058 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T10:24:03,063 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T10:24:03,064 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:24:03,064 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T10:24:03,064 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T10:24:03,067 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T10:24:03,069 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T10:24:03,069 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:03,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:24:03,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T10:24:03,071 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T10:24:03,071 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:03,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:24:03,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T10:24:03,074 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T10:24:03,074 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:03,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:24:03,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T10:24:03,077 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T10:24:03,077 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:03,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:24:03,078 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T10:24:03,079 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740 2024-11-20T10:24:03,082 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740 2024-11-20T10:24:03,084 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T10:24:03,085 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T10:24:03,086 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T10:24:03,088 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T10:24:03,090 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60792900, jitterRate=-0.09411519765853882}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T10:24:03,091 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T10:24:03,093 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732098243065Writing region info on filesystem at 1732098243065Initializing all the Stores at 1732098243067 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098243067Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098243067Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098243067Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098243067Cleaning up temporary data from old regions at 1732098243085 (+18 ms)Running coprocessor post-open hooks at 1732098243091 (+6 ms)Region opened successfully at 1732098243093 (+2 ms) 2024-11-20T10:24:03,100 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732098242999 2024-11-20T10:24:03,111 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T10:24:03,111 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T10:24:03,113 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:24:03,115 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8c533afd331c,35833,1732098238678, state=OPEN 2024-11-20T10:24:03,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T10:24:03,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T10:24:03,216 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:24:03,216 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:24:03,216 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:03,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T10:24:03,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8c533afd331c,35833,1732098238678 in 389 msec 2024-11-20T10:24:03,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T10:24:03,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 670 msec 2024-11-20T10:24:03,234 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T10:24:03,234 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T10:24:03,251 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:03,252 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:03,269 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:03,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47289, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:03,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7470 sec 2024-11-20T10:24:03,294 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732098243294, completionTime=-1 2024-11-20T10:24:03,298 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-20T10:24:03,298 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T10:24:03,346 INFO [master/8c533afd331c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-20T10:24:03,346 INFO [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732098303346 2024-11-20T10:24:03,346 INFO [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732098363346 2024-11-20T10:24:03,346 INFO [master/8c533afd331c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 48 msec 2024-11-20T10:24:03,349 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,33867,1732098237476-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:03,349 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,33867,1732098237476-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:03,349 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,33867,1732098237476-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:03,351 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8c533afd331c:33867, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:03,351 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:03,352 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:03,359 DEBUG [master/8c533afd331c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T10:24:03,377 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.512sec 2024-11-20T10:24:03,378 INFO [master/8c533afd331c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T10:24:03,379 INFO [master/8c533afd331c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T10:24:03,380 INFO [master/8c533afd331c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T10:24:03,381 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T10:24:03,381 INFO [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T10:24:03,381 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,33867,1732098237476-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T10:24:03,382 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,33867,1732098237476-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T10:24:03,389 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T10:24:03,390 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T10:24:03,390 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,33867,1732098237476-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:24:03,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3e3753, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:03,407 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T10:24:03,407 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T10:24:03,411 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:03,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:03,419 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:03,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:03,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb5a2c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:03,430 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:03,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35f487a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:03,433 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:03,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:03,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8c533afd331c,33867,1732098237476 2024-11-20T10:24:03,473 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=9450 2024-11-20T10:24:03,480 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T10:24:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 8c533afd331c,33867,1732098237476 2024-11-20T10:24:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@453fbba3 2024-11-20T10:24:03,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T10:24:03,492 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T10:24:03,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T10:24:03,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T10:24:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T10:24:03,509 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T10:24:03,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-20T10:24:03,511 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:03,513 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T10:24:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T10:24:03,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741835_1011 (size=959) 2024-11-20T10:24:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T10:24:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T10:24:03,930 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31 2024-11-20T10:24:03,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741836_1012 (size=53) 2024-11-20T10:24:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T10:24:04,345 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:24:04,345 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 455386d186b9f64def53c239491339e4, disabling compactions & flushes 2024-11-20T10:24:04,345 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:04,345 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:04,345 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. after waiting 0 ms 2024-11-20T10:24:04,345 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:04,346 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:04,346 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 455386d186b9f64def53c239491339e4: Waiting for close lock at 1732098244345Disabling compacts and flushes for region at 1732098244345Disabling writes for close at 1732098244345Writing region close event to WAL at 1732098244346 (+1 ms)Closed at 1732098244346 2024-11-20T10:24:04,349 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T10:24:04,354 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732098244349"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732098244349"}]},"ts":"1732098244349"} 2024-11-20T10:24:04,359 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T10:24:04,360 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T10:24:04,363 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098244360"}]},"ts":"1732098244360"} 2024-11-20T10:24:04,367 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T10:24:04,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=455386d186b9f64def53c239491339e4, ASSIGN}] 2024-11-20T10:24:04,372 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=455386d186b9f64def53c239491339e4, ASSIGN 2024-11-20T10:24:04,374 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=455386d186b9f64def53c239491339e4, ASSIGN; state=OFFLINE, location=8c533afd331c,35833,1732098238678; forceNewPlan=false, retain=false 2024-11-20T10:24:04,528 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=455386d186b9f64def53c239491339e4, regionState=OPENING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:24:04,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=455386d186b9f64def53c239491339e4, ASSIGN because future has completed 2024-11-20T10:24:04,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:24:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T10:24:04,698 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:04,699 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:24:04,699 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 455386d186b9f64def53c239491339e4 2024-11-20T10:24:04,699 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:24:04,700 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 455386d186b9f64def53c239491339e4 2024-11-20T10:24:04,700 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 455386d186b9f64def53c239491339e4 2024-11-20T10:24:04,703 INFO [StoreOpener-455386d186b9f64def53c239491339e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 455386d186b9f64def53c239491339e4 2024-11-20T10:24:04,706 INFO [StoreOpener-455386d186b9f64def53c239491339e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 455386d186b9f64def53c239491339e4 columnFamilyName A 2024-11-20T10:24:04,706 DEBUG [StoreOpener-455386d186b9f64def53c239491339e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:04,707 INFO [StoreOpener-455386d186b9f64def53c239491339e4-1 {}] regionserver.HStore(327): Store=455386d186b9f64def53c239491339e4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:24:04,708 INFO [StoreOpener-455386d186b9f64def53c239491339e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 455386d186b9f64def53c239491339e4 2024-11-20T10:24:04,710 INFO [StoreOpener-455386d186b9f64def53c239491339e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 455386d186b9f64def53c239491339e4 columnFamilyName B 2024-11-20T10:24:04,710 DEBUG [StoreOpener-455386d186b9f64def53c239491339e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:04,711 INFO [StoreOpener-455386d186b9f64def53c239491339e4-1 {}] regionserver.HStore(327): Store=455386d186b9f64def53c239491339e4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:24:04,711 INFO [StoreOpener-455386d186b9f64def53c239491339e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 455386d186b9f64def53c239491339e4 2024-11-20T10:24:04,713 INFO [StoreOpener-455386d186b9f64def53c239491339e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 455386d186b9f64def53c239491339e4 columnFamilyName C 2024-11-20T10:24:04,713 DEBUG [StoreOpener-455386d186b9f64def53c239491339e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:04,714 INFO [StoreOpener-455386d186b9f64def53c239491339e4-1 {}] regionserver.HStore(327): Store=455386d186b9f64def53c239491339e4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:24:04,714 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 455386d186b9f64def53c239491339e4 2024-11-20T10:24:04,716 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4 2024-11-20T10:24:04,717 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4 2024-11-20T10:24:04,719 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 455386d186b9f64def53c239491339e4 2024-11-20T10:24:04,719 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 455386d186b9f64def53c239491339e4 2024-11-20T10:24:04,720 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T10:24:04,722 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 455386d186b9f64def53c239491339e4 2024-11-20T10:24:04,725 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:24:04,726 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 455386d186b9f64def53c239491339e4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72449863, jitterRate=0.07958708703517914}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T10:24:04,726 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 455386d186b9f64def53c239491339e4 2024-11-20T10:24:04,727 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 455386d186b9f64def53c239491339e4: Running coprocessor pre-open hook at 1732098244700Writing region info on filesystem at 1732098244700Initializing all the Stores at 1732098244702 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098244702Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098244703 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098244703Cleaning up temporary data from old regions at 1732098244719 (+16 ms)Running coprocessor post-open hooks at 1732098244726 (+7 ms)Region opened successfully at 1732098244727 (+1 ms) 2024-11-20T10:24:04,729 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., pid=6, masterSystemTime=1732098244692 2024-11-20T10:24:04,733 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:04,733 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:04,734 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=455386d186b9f64def53c239491339e4, regionState=OPEN, openSeqNum=2, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:24:04,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:24:04,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T10:24:04,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 in 203 msec 2024-11-20T10:24:04,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T10:24:04,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=455386d186b9f64def53c239491339e4, ASSIGN in 375 msec 2024-11-20T10:24:04,750 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T10:24:04,750 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098244750"}]},"ts":"1732098244750"} 2024-11-20T10:24:04,756 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T10:24:04,758 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T10:24:04,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2570 sec 2024-11-20T10:24:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T10:24:05,669 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:05,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed391d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:05,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:05,680 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66c67efa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:05,683 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:05,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48f92cdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,685 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:05,690 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:05,692 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T10:24:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8c533afd331c,33867,1732098237476 2024-11-20T10:24:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@e6a8148 2024-11-20T10:24:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T10:24:05,699 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T10:24:05,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e6c3037, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,703 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:05,703 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:05,705 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14496fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:05,709 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e49c5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a519624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,711 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:05,711 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:05,712 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f4c9506, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:05,716 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36644156, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa662a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,719 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:05,719 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:05,721 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:05,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:05,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:05,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a9c1aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:05,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:05,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:05,725 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:05,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3937ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a38e5f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,727 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:05,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:05,729 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:05,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:05,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:05,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffd0298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:05,733 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@615aad25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658d066f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,736 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:05,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:05,737 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:05,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:05,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b904aa3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:05,739 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:05,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df19f74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d220de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:05,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:05,746 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:05,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:05,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a6891f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:05,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:05,749 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe9eb0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32cda1eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:05,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:05,752 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@739b8c1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:05,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:05,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:05,755 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ce29b6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d59c73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:05,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:05,759 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:05,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:05,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19ac4850, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:05,761 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:05,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44b83c21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27451c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:05,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:05,765 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:05,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68732d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:05,768 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:05,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ce716ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:05,772 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:05,772 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:05,772 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:05,773 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:05,774 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:05,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:05,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:05,775 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:05,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:05,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:05,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:05,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:05,778 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:05,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:05,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:05,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:05,779 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:05,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:05,782 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:05,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:05,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:05,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:05,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:24:05,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:24:05,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:24:05,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:24:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:24:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:24:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:24:05,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:24:05,795 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-20T10:24:05,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:05,802 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:05,803 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T10:24:05,804 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:05,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:05,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:24:05,878 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:05,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:24:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T10:24:05,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T10:24:06,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:06,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:06,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/bf352c7c2c754fb693673e87b5e880f7 is 50, key is test_row_0/A:col10/1732098245865/Put/seqid=0 2024-11-20T10:24:06,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:06,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741837_1013 (size=14341) 2024-11-20T10:24:06,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/bf352c7c2c754fb693673e87b5e880f7 2024-11-20T10:24:06,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098306063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098306064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098306056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098306068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098306075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T10:24:06,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T10:24:06,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/1fc4a435afc1417493de4da4dd282e6f is 50, key is test_row_0/B:col10/1732098245865/Put/seqid=0 2024-11-20T10:24:06,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:06,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:06,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:06,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741838_1014 (size=12001) 2024-11-20T10:24:06,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/1fc4a435afc1417493de4da4dd282e6f 2024-11-20T10:24:06,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098306237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098306239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098306238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098306243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098306245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/95167c307c1b4509bf4ad79896653625 is 50, key is test_row_0/C:col10/1732098245865/Put/seqid=0 2024-11-20T10:24:06,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741839_1015 (size=12001) 2024-11-20T10:24:06,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T10:24:06,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:06,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:06,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:06,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T10:24:06,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098306447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098306460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098306460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098306461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098306461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T10:24:06,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:06,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:06,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:06,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T10:24:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:06,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:06,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:06,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/95167c307c1b4509bf4ad79896653625 2024-11-20T10:24:06,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/bf352c7c2c754fb693673e87b5e880f7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bf352c7c2c754fb693673e87b5e880f7 2024-11-20T10:24:06,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bf352c7c2c754fb693673e87b5e880f7, entries=200, sequenceid=13, filesize=14.0 K 2024-11-20T10:24:06,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/1fc4a435afc1417493de4da4dd282e6f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1fc4a435afc1417493de4da4dd282e6f 2024-11-20T10:24:06,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1fc4a435afc1417493de4da4dd282e6f, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T10:24:06,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/95167c307c1b4509bf4ad79896653625 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/95167c307c1b4509bf4ad79896653625 2024-11-20T10:24:06,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098306757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098306778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/95167c307c1b4509bf4ad79896653625, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T10:24:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 455386d186b9f64def53c239491339e4 in 911ms, sequenceid=13, compaction requested=false 2024-11-20T10:24:06,804 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T10:24:06,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:06,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-20T10:24:06,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T10:24:06,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:06,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:06,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:06,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098306849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/bd7ae4f3ac724c45897fb6d97e265b32 is 50, key is test_row_0/A:col10/1732098246054/Put/seqid=0 2024-11-20T10:24:06,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098306864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098306871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741840_1016 (size=21365) 2024-11-20T10:24:06,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/bd7ae4f3ac724c45897fb6d97e265b32 2024-11-20T10:24:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T10:24:06,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/1f57febc9b8646dc9a54dd13f3cf8638 is 50, key is test_row_0/B:col10/1732098246054/Put/seqid=0 2024-11-20T10:24:06,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098306978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:06,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T10:24:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:06,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:06,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741841_1017 (size=12001) 2024-11-20T10:24:07,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/1f57febc9b8646dc9a54dd13f3cf8638 2024-11-20T10:24:07,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098307018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098307018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/9efe30daff6145768a54524b49b207b5 is 50, key is test_row_0/C:col10/1732098246054/Put/seqid=0 2024-11-20T10:24:07,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741842_1018 (size=12001) 2024-11-20T10:24:07,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/9efe30daff6145768a54524b49b207b5 2024-11-20T10:24:07,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/bd7ae4f3ac724c45897fb6d97e265b32 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bd7ae4f3ac724c45897fb6d97e265b32 2024-11-20T10:24:07,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bd7ae4f3ac724c45897fb6d97e265b32, entries=350, sequenceid=41, filesize=20.9 K 2024-11-20T10:24:07,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/1f57febc9b8646dc9a54dd13f3cf8638 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1f57febc9b8646dc9a54dd13f3cf8638 2024-11-20T10:24:07,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1f57febc9b8646dc9a54dd13f3cf8638, entries=150, sequenceid=41, filesize=11.7 K 2024-11-20T10:24:07,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/9efe30daff6145768a54524b49b207b5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9efe30daff6145768a54524b49b207b5 2024-11-20T10:24:07,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9efe30daff6145768a54524b49b207b5, entries=150, sequenceid=41, filesize=11.7 K 2024-11-20T10:24:07,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T10:24:07,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:07,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:07,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:07,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:07,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:07,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 455386d186b9f64def53c239491339e4 in 339ms, sequenceid=41, compaction requested=false 2024-11-20T10:24:07,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:07,234 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:07,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:24:07,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/12743c97e39744f2a2e7e2500847ba5c is 50, key is test_row_0/A:col10/1732098246849/Put/seqid=0 2024-11-20T10:24:07,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741843_1019 (size=16681) 2024-11-20T10:24:07,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/12743c97e39744f2a2e7e2500847ba5c 2024-11-20T10:24:07,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T10:24:07,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:07,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:07,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:07,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:07,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:07,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/e8b19409ce6b4c7582cc25ff06d5d365 is 50, key is test_row_0/B:col10/1732098246849/Put/seqid=0 2024-11-20T10:24:07,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741844_1020 (size=12001) 2024-11-20T10:24:07,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/e8b19409ce6b4c7582cc25ff06d5d365 2024-11-20T10:24:07,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098307344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098307347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098307362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098307365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/318508ad74f04099824e93fe3fd61e79 is 50, key is test_row_0/C:col10/1732098246849/Put/seqid=0 2024-11-20T10:24:07,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098307366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741845_1021 (size=12001) 2024-11-20T10:24:07,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/318508ad74f04099824e93fe3fd61e79 2024-11-20T10:24:07,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/12743c97e39744f2a2e7e2500847ba5c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/12743c97e39744f2a2e7e2500847ba5c 2024-11-20T10:24:07,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T10:24:07,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/12743c97e39744f2a2e7e2500847ba5c, entries=250, sequenceid=52, filesize=16.3 K 2024-11-20T10:24:07,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:07,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:07,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:07,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:07,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:07,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/e8b19409ce6b4c7582cc25ff06d5d365 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/e8b19409ce6b4c7582cc25ff06d5d365 2024-11-20T10:24:07,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098307478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098307478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098307488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098307489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/e8b19409ce6b4c7582cc25ff06d5d365, entries=150, sequenceid=52, filesize=11.7 K 2024-11-20T10:24:07,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098307497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/318508ad74f04099824e93fe3fd61e79 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/318508ad74f04099824e93fe3fd61e79 2024-11-20T10:24:07,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/318508ad74f04099824e93fe3fd61e79, entries=150, sequenceid=52, filesize=11.7 K 2024-11-20T10:24:07,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 455386d186b9f64def53c239491339e4 in 295ms, sequenceid=52, compaction requested=true 2024-11-20T10:24:07,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:07,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:07,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:07,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:07,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:07,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:07,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:07,543 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:07,548 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:07,549 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:07,553 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/B is initiating minor compaction (all files) 2024-11-20T10:24:07,557 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/B in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:07,557 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1fc4a435afc1417493de4da4dd282e6f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1f57febc9b8646dc9a54dd13f3cf8638, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/e8b19409ce6b4c7582cc25ff06d5d365] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=35.2 K 2024-11-20T10:24:07,560 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 52387 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:07,560 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/A is initiating minor compaction (all files) 2024-11-20T10:24:07,561 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/A in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:07,561 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bf352c7c2c754fb693673e87b5e880f7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bd7ae4f3ac724c45897fb6d97e265b32, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/12743c97e39744f2a2e7e2500847ba5c] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=51.2 K 2024-11-20T10:24:07,564 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf352c7c2c754fb693673e87b5e880f7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732098245858 2024-11-20T10:24:07,567 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fc4a435afc1417493de4da4dd282e6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732098245860 2024-11-20T10:24:07,568 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f57febc9b8646dc9a54dd13f3cf8638, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732098246054 2024-11-20T10:24:07,569 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd7ae4f3ac724c45897fb6d97e265b32, keycount=350, bloomtype=ROW, size=20.9 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732098246054 2024-11-20T10:24:07,569 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting e8b19409ce6b4c7582cc25ff06d5d365, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732098246849 2024-11-20T10:24:07,569 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12743c97e39744f2a2e7e2500847ba5c, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732098246849 2024-11-20T10:24:07,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T10:24:07,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:07,622 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#A#compaction#9 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:07,622 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T10:24:07,623 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/1d6b77f46a6e473481a12d75f57bdd17 is 50, key is test_row_0/A:col10/1732098246849/Put/seqid=0 2024-11-20T10:24:07,624 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#B#compaction#10 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:07,625 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/eadaa23c704349cf95a92eabfd649dce is 50, key is test_row_0/B:col10/1732098246849/Put/seqid=0 2024-11-20T10:24:07,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/bd40961b524b4b5c98d4d530a63c1b2c is 50, key is test_row_0/A:col10/1732098247360/Put/seqid=0 2024-11-20T10:24:07,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741846_1022 (size=12104) 2024-11-20T10:24:07,682 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/1d6b77f46a6e473481a12d75f57bdd17 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/1d6b77f46a6e473481a12d75f57bdd17 2024-11-20T10:24:07,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741848_1024 (size=12001) 2024-11-20T10:24:07,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:07,701 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/bd40961b524b4b5c98d4d530a63c1b2c 2024-11-20T10:24:07,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741847_1023 (size=12104) 2024-11-20T10:24:07,736 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/A of 455386d186b9f64def53c239491339e4 into 1d6b77f46a6e473481a12d75f57bdd17(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:07,736 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:07,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/596c1778a141454da82eac3f8a806b1f is 50, key is test_row_0/B:col10/1732098247360/Put/seqid=0 2024-11-20T10:24:07,742 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/A, priority=13, startTime=1732098247532; duration=0sec 2024-11-20T10:24:07,742 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:07,743 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:A 2024-11-20T10:24:07,743 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:07,745 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:07,745 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/C is initiating minor compaction (all files) 2024-11-20T10:24:07,746 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/C in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:07,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098307722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,750 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/95167c307c1b4509bf4ad79896653625, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9efe30daff6145768a54524b49b207b5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/318508ad74f04099824e93fe3fd61e79] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=35.2 K 2024-11-20T10:24:07,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098307723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098307731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,755 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95167c307c1b4509bf4ad79896653625, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732098245860 2024-11-20T10:24:07,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,756 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9efe30daff6145768a54524b49b207b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732098246054 2024-11-20T10:24:07,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098307734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,756 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/eadaa23c704349cf95a92eabfd649dce as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/eadaa23c704349cf95a92eabfd649dce 2024-11-20T10:24:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098307740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,760 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 318508ad74f04099824e93fe3fd61e79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732098246849 2024-11-20T10:24:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,774 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/B of 455386d186b9f64def53c239491339e4 into eadaa23c704349cf95a92eabfd649dce(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:07,774 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:07,774 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/B, priority=13, startTime=1732098247542; duration=0sec 2024-11-20T10:24:07,774 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:07,774 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:B 2024-11-20T10:24:07,786 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T10:24:07,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741849_1025 (size=12001) 2024-11-20T10:24:07,792 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/596c1778a141454da82eac3f8a806b1f 2024-11-20T10:24:07,801 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#C#compaction#13 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:07,802 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/9849114ada404d7b82551d825f09d772 is 50, key is test_row_0/C:col10/1732098246849/Put/seqid=0 2024-11-20T10:24:07,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/1ee1378de1304428a3c32f1c35d642ef is 50, key is test_row_0/C:col10/1732098247360/Put/seqid=0 2024-11-20T10:24:07,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741850_1026 (size=12104) 2024-11-20T10:24:07,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098307857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098307858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098307860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741851_1027 (size=12001) 2024-11-20T10:24:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,865 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/1ee1378de1304428a3c32f1c35d642ef 2024-11-20T10:24:07,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098307867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:07,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098307867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:07,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:07,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/bd40961b524b4b5c98d4d530a63c1b2c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bd40961b524b4b5c98d4d530a63c1b2c 2024-11-20T10:24:07,888 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T10:24:07,888 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T10:24:07,896 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T10:24:07,896 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T10:24:07,897 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bd40961b524b4b5c98d4d530a63c1b2c, entries=150, sequenceid=78, filesize=11.7 K 2024-11-20T10:24:07,897 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T10:24:07,897 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T10:24:07,899 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T10:24:07,899 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T10:24:07,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/596c1778a141454da82eac3f8a806b1f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/596c1778a141454da82eac3f8a806b1f 2024-11-20T10:24:07,921 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/596c1778a141454da82eac3f8a806b1f, entries=150, sequenceid=78, filesize=11.7 K 2024-11-20T10:24:07,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/1ee1378de1304428a3c32f1c35d642ef as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1ee1378de1304428a3c32f1c35d642ef 2024-11-20T10:24:07,944 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1ee1378de1304428a3c32f1c35d642ef, entries=150, sequenceid=78, filesize=11.7 K 2024-11-20T10:24:07,946 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 455386d186b9f64def53c239491339e4 in 324ms, sequenceid=78, compaction requested=false 2024-11-20T10:24:07,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:07,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:07,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-20T10:24:07,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T10:24:07,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-20T10:24:07,952 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T10:24:07,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1430 sec 2024-11-20T10:24:07,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.1730 sec 2024-11-20T10:24:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:08,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:24:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:08,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/52a2983ea55d4ad0a4cbff64f72b3d6c is 50, key is test_row_0/A:col10/1732098248075/Put/seqid=0 2024-11-20T10:24:08,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741852_1028 (size=16681) 2024-11-20T10:24:08,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/52a2983ea55d4ad0a4cbff64f72b3d6c 2024-11-20T10:24:08,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/cb83d9ee9cb24fbbae0366ba580a64ac is 50, key is test_row_0/B:col10/1732098248075/Put/seqid=0 2024-11-20T10:24:08,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741853_1029 (size=12001) 2024-11-20T10:24:08,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/cb83d9ee9cb24fbbae0366ba580a64ac 2024-11-20T10:24:08,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/70d18833a5f142749e6cd3f09fbc571b is 50, key is test_row_0/C:col10/1732098248075/Put/seqid=0 2024-11-20T10:24:08,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741854_1030 (size=12001) 2024-11-20T10:24:08,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/70d18833a5f142749e6cd3f09fbc571b 2024-11-20T10:24:08,296 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/9849114ada404d7b82551d825f09d772 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9849114ada404d7b82551d825f09d772 2024-11-20T10:24:08,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/52a2983ea55d4ad0a4cbff64f72b3d6c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/52a2983ea55d4ad0a4cbff64f72b3d6c 2024-11-20T10:24:08,319 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/C of 455386d186b9f64def53c239491339e4 into 9849114ada404d7b82551d825f09d772(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:08,319 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:08,320 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/C, priority=13, startTime=1732098247542; duration=0sec 2024-11-20T10:24:08,320 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:08,320 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:C 2024-11-20T10:24:08,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/52a2983ea55d4ad0a4cbff64f72b3d6c, entries=250, sequenceid=92, filesize=16.3 K 2024-11-20T10:24:08,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/cb83d9ee9cb24fbbae0366ba580a64ac as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/cb83d9ee9cb24fbbae0366ba580a64ac 2024-11-20T10:24:08,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098308317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098308318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098308323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098308343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098308346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/cb83d9ee9cb24fbbae0366ba580a64ac, entries=150, sequenceid=92, filesize=11.7 K 2024-11-20T10:24:08,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/70d18833a5f142749e6cd3f09fbc571b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/70d18833a5f142749e6cd3f09fbc571b 2024-11-20T10:24:08,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/70d18833a5f142749e6cd3f09fbc571b, entries=150, sequenceid=92, filesize=11.7 K 2024-11-20T10:24:08,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 455386d186b9f64def53c239491339e4 in 309ms, sequenceid=92, compaction requested=true 2024-11-20T10:24:08,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:08,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:08,387 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:08,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:08,387 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:08,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:08,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:08,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:08,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:08,390 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:08,390 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:08,390 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/B is initiating minor compaction (all files) 2024-11-20T10:24:08,390 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/A is initiating minor compaction (all files) 2024-11-20T10:24:08,390 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/B in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:08,390 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/A in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:08,391 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/eadaa23c704349cf95a92eabfd649dce, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/596c1778a141454da82eac3f8a806b1f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/cb83d9ee9cb24fbbae0366ba580a64ac] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=35.3 K 2024-11-20T10:24:08,391 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/1d6b77f46a6e473481a12d75f57bdd17, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bd40961b524b4b5c98d4d530a63c1b2c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/52a2983ea55d4ad0a4cbff64f72b3d6c] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=39.8 K 2024-11-20T10:24:08,392 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting eadaa23c704349cf95a92eabfd649dce, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732098246849 2024-11-20T10:24:08,392 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d6b77f46a6e473481a12d75f57bdd17, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732098246849 2024-11-20T10:24:08,393 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd40961b524b4b5c98d4d530a63c1b2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732098247333 2024-11-20T10:24:08,393 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 596c1778a141454da82eac3f8a806b1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732098247333 2024-11-20T10:24:08,394 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting cb83d9ee9cb24fbbae0366ba580a64ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732098247723 2024-11-20T10:24:08,394 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52a2983ea55d4ad0a4cbff64f72b3d6c, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732098247722 2024-11-20T10:24:08,418 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#A#compaction#18 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:08,420 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/9165702575de41ff8933c63c310a2d91 is 50, key is test_row_0/A:col10/1732098248075/Put/seqid=0 2024-11-20T10:24:08,443 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:08,444 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/9cb3ad7e5ba045a190804a580489ac44 is 50, key is test_row_0/B:col10/1732098248075/Put/seqid=0 2024-11-20T10:24:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:08,461 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:08,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T10:24:08,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741855_1031 (size=12207) 2024-11-20T10:24:08,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098308475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,492 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/9165702575de41ff8933c63c310a2d91 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/9165702575de41ff8933c63c310a2d91 2024-11-20T10:24:08,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741856_1032 (size=12207) 2024-11-20T10:24:08,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/8a55b949c8674cc698ce7516449d2ab2 is 50, key is test_row_0/A:col10/1732098248309/Put/seqid=0 2024-11-20T10:24:08,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098308490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098308491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,510 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/9cb3ad7e5ba045a190804a580489ac44 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/9cb3ad7e5ba045a190804a580489ac44 2024-11-20T10:24:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098308492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098308495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,514 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/A of 455386d186b9f64def53c239491339e4 into 9165702575de41ff8933c63c310a2d91(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:08,515 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:08,515 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/A, priority=13, startTime=1732098248387; duration=0sec 2024-11-20T10:24:08,515 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:08,515 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:A 2024-11-20T10:24:08,515 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,520 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:08,521 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/C is initiating minor compaction (all files) 2024-11-20T10:24:08,526 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/C in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:08,526 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9849114ada404d7b82551d825f09d772, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1ee1378de1304428a3c32f1c35d642ef, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/70d18833a5f142749e6cd3f09fbc571b] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=35.3 K 2024-11-20T10:24:08,528 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/B of 455386d186b9f64def53c239491339e4 into 9cb3ad7e5ba045a190804a580489ac44(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:08,528 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:08,528 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/B, priority=13, startTime=1732098248387; duration=0sec 2024-11-20T10:24:08,529 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:08,529 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:B 2024-11-20T10:24:08,529 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9849114ada404d7b82551d825f09d772, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732098246849 2024-11-20T10:24:08,530 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ee1378de1304428a3c32f1c35d642ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732098247333 2024-11-20T10:24:08,532 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70d18833a5f142749e6cd3f09fbc571b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732098247723 2024-11-20T10:24:08,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741857_1033 (size=12001) 2024-11-20T10:24:08,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/8a55b949c8674cc698ce7516449d2ab2 2024-11-20T10:24:08,565 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:08,566 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/09c9d5458f2f44e0b2eddef0824fb9ed is 50, key is test_row_0/C:col10/1732098248075/Put/seqid=0 2024-11-20T10:24:08,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/bf0edc5087584ad2a5f641a4c736f92d is 50, key is test_row_0/B:col10/1732098248309/Put/seqid=0 2024-11-20T10:24:08,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,599 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T10:24:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098308597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741858_1034 (size=12207) 2024-11-20T10:24:08,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098308617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098308617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741859_1035 (size=12001) 2024-11-20T10:24:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098308617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098308627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,642 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/09c9d5458f2f44e0b2eddef0824fb9ed as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/09c9d5458f2f44e0b2eddef0824fb9ed 2024-11-20T10:24:08,665 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/C of 455386d186b9f64def53c239491339e4 into 09c9d5458f2f44e0b2eddef0824fb9ed(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:08,665 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:08,665 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/C, priority=13, startTime=1732098248387; duration=0sec 2024-11-20T10:24:08,665 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:08,665 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:C 2024-11-20T10:24:08,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098308807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098308827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098308827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098308827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:08,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098308837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/bf0edc5087584ad2a5f641a4c736f92d 2024-11-20T10:24:09,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/1b636f12a83a425b9eb4a729d24033f2 is 50, key is test_row_0/C:col10/1732098248309/Put/seqid=0 2024-11-20T10:24:09,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741860_1036 (size=12001) 2024-11-20T10:24:09,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/1b636f12a83a425b9eb4a729d24033f2 2024-11-20T10:24:09,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/8a55b949c8674cc698ce7516449d2ab2 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/8a55b949c8674cc698ce7516449d2ab2 2024-11-20T10:24:09,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/8a55b949c8674cc698ce7516449d2ab2, entries=150, sequenceid=119, filesize=11.7 K 2024-11-20T10:24:09,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/bf0edc5087584ad2a5f641a4c736f92d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/bf0edc5087584ad2a5f641a4c736f92d 2024-11-20T10:24:09,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/bf0edc5087584ad2a5f641a4c736f92d, entries=150, sequenceid=119, filesize=11.7 K 2024-11-20T10:24:09,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098309117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/1b636f12a83a425b9eb4a729d24033f2 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1b636f12a83a425b9eb4a729d24033f2 2024-11-20T10:24:09,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1b636f12a83a425b9eb4a729d24033f2, entries=150, sequenceid=119, filesize=11.7 K 2024-11-20T10:24:09,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 455386d186b9f64def53c239491339e4 in 679ms, sequenceid=119, compaction requested=false 2024-11-20T10:24:09,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:09,141 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:09,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T10:24:09,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/ee495d7befe04de490cf7bf2575d08bd is 50, key is test_row_0/A:col10/1732098249137/Put/seqid=0 2024-11-20T10:24:09,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741861_1037 (size=12101) 2024-11-20T10:24:09,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/ee495d7befe04de490cf7bf2575d08bd 2024-11-20T10:24:09,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098309208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098309209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098309211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098309211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/c2b41641be0f423f90f6e37bf6a4b576 is 50, key is test_row_0/B:col10/1732098249137/Put/seqid=0 2024-11-20T10:24:09,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741862_1038 (size=12101) 2024-11-20T10:24:09,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/c2b41641be0f423f90f6e37bf6a4b576 2024-11-20T10:24:09,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/babc86afd0eb4adfb8b48bad5d19ba1d is 50, key is test_row_0/C:col10/1732098249137/Put/seqid=0 2024-11-20T10:24:09,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741863_1039 (size=12101) 2024-11-20T10:24:09,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/babc86afd0eb4adfb8b48bad5d19ba1d 2024-11-20T10:24:09,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098309327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098309327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098309327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098309327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/ee495d7befe04de490cf7bf2575d08bd as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/ee495d7befe04de490cf7bf2575d08bd 2024-11-20T10:24:09,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/ee495d7befe04de490cf7bf2575d08bd, entries=150, sequenceid=135, filesize=11.8 K 2024-11-20T10:24:09,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/c2b41641be0f423f90f6e37bf6a4b576 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c2b41641be0f423f90f6e37bf6a4b576 2024-11-20T10:24:09,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c2b41641be0f423f90f6e37bf6a4b576, entries=150, sequenceid=135, filesize=11.8 K 2024-11-20T10:24:09,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/babc86afd0eb4adfb8b48bad5d19ba1d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/babc86afd0eb4adfb8b48bad5d19ba1d 2024-11-20T10:24:09,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/babc86afd0eb4adfb8b48bad5d19ba1d, entries=150, sequenceid=135, filesize=11.8 K 2024-11-20T10:24:09,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 455386d186b9f64def53c239491339e4 in 238ms, sequenceid=135, compaction requested=true 2024-11-20T10:24:09,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:09,380 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:09,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:09,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:09,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:09,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:09,381 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:09,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:09,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:09,384 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:09,384 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/A is initiating minor compaction (all files) 2024-11-20T10:24:09,385 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/A in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:09,385 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/9165702575de41ff8933c63c310a2d91, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/8a55b949c8674cc698ce7516449d2ab2, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/ee495d7befe04de490cf7bf2575d08bd] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=35.5 K 2024-11-20T10:24:09,385 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:09,386 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9165702575de41ff8933c63c310a2d91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732098247723 2024-11-20T10:24:09,386 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/B is initiating minor compaction (all files) 2024-11-20T10:24:09,386 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/B in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:09,386 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/9cb3ad7e5ba045a190804a580489ac44, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/bf0edc5087584ad2a5f641a4c736f92d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c2b41641be0f423f90f6e37bf6a4b576] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=35.5 K 2024-11-20T10:24:09,387 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a55b949c8674cc698ce7516449d2ab2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732098248309 2024-11-20T10:24:09,388 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cb3ad7e5ba045a190804a580489ac44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732098247723 2024-11-20T10:24:09,388 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ee495d7befe04de490cf7bf2575d08bd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732098248487 2024-11-20T10:24:09,389 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf0edc5087584ad2a5f641a4c736f92d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732098248309 2024-11-20T10:24:09,390 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2b41641be0f423f90f6e37bf6a4b576, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732098248487 2024-11-20T10:24:09,420 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:09,421 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/4559e6f10b51442c94324110bb36b3fb is 50, key is test_row_0/B:col10/1732098249137/Put/seqid=0 2024-11-20T10:24:09,435 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:09,436 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/2e5143a4f1b440b2bc70cd7525565435 is 50, key is test_row_0/A:col10/1732098249137/Put/seqid=0 2024-11-20T10:24:09,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741865_1041 (size=12409) 2024-11-20T10:24:09,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741864_1040 (size=12409) 2024-11-20T10:24:09,501 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/2e5143a4f1b440b2bc70cd7525565435 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2e5143a4f1b440b2bc70cd7525565435 2024-11-20T10:24:09,510 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/4559e6f10b51442c94324110bb36b3fb as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/4559e6f10b51442c94324110bb36b3fb 2024-11-20T10:24:09,515 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/A of 455386d186b9f64def53c239491339e4 into 2e5143a4f1b440b2bc70cd7525565435(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:09,515 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:09,515 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/A, priority=13, startTime=1732098249380; duration=0sec 2024-11-20T10:24:09,516 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:09,516 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:A 2024-11-20T10:24:09,516 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:09,519 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:09,519 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/C is initiating minor compaction (all files) 2024-11-20T10:24:09,519 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/C in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:09,520 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/09c9d5458f2f44e0b2eddef0824fb9ed, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1b636f12a83a425b9eb4a729d24033f2, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/babc86afd0eb4adfb8b48bad5d19ba1d] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=35.5 K 2024-11-20T10:24:09,521 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 09c9d5458f2f44e0b2eddef0824fb9ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732098247723 2024-11-20T10:24:09,521 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/B of 455386d186b9f64def53c239491339e4 into 4559e6f10b51442c94324110bb36b3fb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:09,522 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:09,522 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/B, priority=13, startTime=1732098249381; duration=0sec 2024-11-20T10:24:09,522 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:09,522 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:B 2024-11-20T10:24:09,525 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b636f12a83a425b9eb4a729d24033f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732098248309 2024-11-20T10:24:09,526 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting babc86afd0eb4adfb8b48bad5d19ba1d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732098248487 2024-11-20T10:24:09,547 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#C#compaction#29 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:09,548 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/fd976d3d3fc14906af51d145886a6106 is 50, key is test_row_0/C:col10/1732098249137/Put/seqid=0 2024-11-20T10:24:09,555 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:09,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T10:24:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:09,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741866_1042 (size=12409) 2024-11-20T10:24:09,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/0b730db08cdb466989c53e76a2251b5f is 50, key is test_row_0/A:col10/1732098249552/Put/seqid=0 2024-11-20T10:24:09,600 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/fd976d3d3fc14906af51d145886a6106 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/fd976d3d3fc14906af51d145886a6106 2024-11-20T10:24:09,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098309586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098309591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098309602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098309604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,619 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/C of 455386d186b9f64def53c239491339e4 into fd976d3d3fc14906af51d145886a6106(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:09,619 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:09,619 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/C, priority=13, startTime=1732098249381; duration=0sec 2024-11-20T10:24:09,619 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:09,619 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:C 2024-11-20T10:24:09,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098309627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741867_1043 (size=12151) 2024-11-20T10:24:09,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/0b730db08cdb466989c53e76a2251b5f 2024-11-20T10:24:09,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/36d217b0bf57431e89618aaded207bfe is 50, key is test_row_0/B:col10/1732098249552/Put/seqid=0 2024-11-20T10:24:09,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741868_1044 (size=12151) 2024-11-20T10:24:09,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098309707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098309717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098309718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098309718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098309917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098309927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098309927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098309928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T10:24:09,968 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:09,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-20T10:24:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T10:24:09,974 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:09,977 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:09,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T10:24:10,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/36d217b0bf57431e89618aaded207bfe 2024-11-20T10:24:10,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/4600620715e249d58170d1100fd2bb24 is 50, key is test_row_0/C:col10/1732098249552/Put/seqid=0 2024-11-20T10:24:10,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741869_1045 (size=12151) 2024-11-20T10:24:10,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/4600620715e249d58170d1100fd2bb24 2024-11-20T10:24:10,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/0b730db08cdb466989c53e76a2251b5f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/0b730db08cdb466989c53e76a2251b5f 2024-11-20T10:24:10,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T10:24:10,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:10,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:10,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:10,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:10,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:10,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:10,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/0b730db08cdb466989c53e76a2251b5f, entries=150, sequenceid=162, filesize=11.9 K 2024-11-20T10:24:10,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/36d217b0bf57431e89618aaded207bfe as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/36d217b0bf57431e89618aaded207bfe 2024-11-20T10:24:10,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/36d217b0bf57431e89618aaded207bfe, entries=150, sequenceid=162, filesize=11.9 K 2024-11-20T10:24:10,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/4600620715e249d58170d1100fd2bb24 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/4600620715e249d58170d1100fd2bb24 2024-11-20T10:24:10,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/4600620715e249d58170d1100fd2bb24, entries=150, sequenceid=162, filesize=11.9 K 2024-11-20T10:24:10,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 455386d186b9f64def53c239491339e4 in 619ms, sequenceid=162, compaction requested=false 2024-11-20T10:24:10,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:10,232 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:10,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T10:24:10,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/52295b9bdb0b4906800417b292bfabcc is 50, key is test_row_0/A:col10/1732098250228/Put/seqid=0 2024-11-20T10:24:10,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741870_1046 (size=12151) 2024-11-20T10:24:10,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/52295b9bdb0b4906800417b292bfabcc 2024-11-20T10:24:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T10:24:10,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/b113bda782424af08077f261ce7ba480 is 50, key is test_row_0/B:col10/1732098250228/Put/seqid=0 2024-11-20T10:24:10,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T10:24:10,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:10,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:10,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:10,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:10,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:10,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:10,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098310301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098310303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:10,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098310304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:10,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098310313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:10,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:10,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741871_1047 (size=12151) 2024-11-20T10:24:10,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/b113bda782424af08077f261ce7ba480 2024-11-20T10:24:10,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/8a66abdca7434301b86e42ffc0796855 is 50, key is test_row_0/C:col10/1732098250228/Put/seqid=0 2024-11-20T10:24:10,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741872_1048 (size=12151) 2024-11-20T10:24:10,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098310417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:10,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098310417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:10,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098310417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:10,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098310427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:10,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T10:24:10,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:10,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:10,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:10,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:10,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T10:24:10,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T10:24:10,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:10,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:10,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:10,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:10,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:10,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098310627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:10,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098310637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:10,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:10,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098310637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:10,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:10,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098310637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:10,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098310657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:10,668 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in 455386d186b9f64def53c239491339e4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:24:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:10,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T10:24:10,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:10,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:10,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:10,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:10,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:10,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:10,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/8a66abdca7434301b86e42ffc0796855 2024-11-20T10:24:10,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/52295b9bdb0b4906800417b292bfabcc as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/52295b9bdb0b4906800417b292bfabcc 2024-11-20T10:24:10,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/52295b9bdb0b4906800417b292bfabcc, entries=150, sequenceid=176, filesize=11.9 K 2024-11-20T10:24:10,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/b113bda782424af08077f261ce7ba480 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/b113bda782424af08077f261ce7ba480 2024-11-20T10:24:10,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/b113bda782424af08077f261ce7ba480, entries=150, sequenceid=176, filesize=11.9 K 2024-11-20T10:24:10,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/8a66abdca7434301b86e42ffc0796855 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/8a66abdca7434301b86e42ffc0796855 2024-11-20T10:24:10,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/8a66abdca7434301b86e42ffc0796855, entries=150, sequenceid=176, filesize=11.9 K 2024-11-20T10:24:10,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 455386d186b9f64def53c239491339e4 in 597ms, sequenceid=176, compaction requested=true 2024-11-20T10:24:10,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:10,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:10,830 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:10,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:10,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:10,830 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:10,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:10,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:10,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:10,834 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:10,834 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:10,834 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/A is initiating minor compaction (all files) 2024-11-20T10:24:10,834 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/B is initiating minor compaction (all files) 2024-11-20T10:24:10,835 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/B in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:10,835 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/A in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:10,835 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2e5143a4f1b440b2bc70cd7525565435, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/0b730db08cdb466989c53e76a2251b5f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/52295b9bdb0b4906800417b292bfabcc] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=35.9 K 2024-11-20T10:24:10,835 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/4559e6f10b51442c94324110bb36b3fb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/36d217b0bf57431e89618aaded207bfe, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/b113bda782424af08077f261ce7ba480] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=35.9 K 2024-11-20T10:24:10,836 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e5143a4f1b440b2bc70cd7525565435, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732098248487 2024-11-20T10:24:10,836 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4559e6f10b51442c94324110bb36b3fb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732098248487 2024-11-20T10:24:10,837 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 36d217b0bf57431e89618aaded207bfe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732098249207 2024-11-20T10:24:10,837 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b730db08cdb466989c53e76a2251b5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732098249207 2024-11-20T10:24:10,839 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52295b9bdb0b4906800417b292bfabcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732098249590 2024-11-20T10:24:10,840 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b113bda782424af08077f261ce7ba480, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732098249590 2024-11-20T10:24:10,872 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:10,873 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/a6a3ba3c4ab84c2493438e1010d631cd is 50, key is test_row_0/B:col10/1732098250228/Put/seqid=0 2024-11-20T10:24:10,882 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#A#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:10,883 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/f19fa2b510034e249bd76705212e939c is 50, key is test_row_0/A:col10/1732098250228/Put/seqid=0 2024-11-20T10:24:10,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741873_1049 (size=12561) 2024-11-20T10:24:10,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741874_1050 (size=12561) 2024-11-20T10:24:10,918 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/f19fa2b510034e249bd76705212e939c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/f19fa2b510034e249bd76705212e939c 2024-11-20T10:24:10,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T10:24:10,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:10,922 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T10:24:10,929 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/A of 455386d186b9f64def53c239491339e4 into f19fa2b510034e249bd76705212e939c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:10,929 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:10,929 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/A, priority=13, startTime=1732098250830; duration=0sec 2024-11-20T10:24:10,930 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:10,930 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:A 2024-11-20T10:24:10,930 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:10,932 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:10,932 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/C is initiating minor compaction (all files) 2024-11-20T10:24:10,932 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/C in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:10,932 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/fd976d3d3fc14906af51d145886a6106, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/4600620715e249d58170d1100fd2bb24, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/8a66abdca7434301b86e42ffc0796855] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=35.9 K 2024-11-20T10:24:10,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/a940aa4b46a1408d9d374a0fd8ab9b6f is 50, key is test_row_0/A:col10/1732098250283/Put/seqid=0 2024-11-20T10:24:10,935 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd976d3d3fc14906af51d145886a6106, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732098248487 2024-11-20T10:24:10,936 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4600620715e249d58170d1100fd2bb24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732098249207 2024-11-20T10:24:10,936 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a66abdca7434301b86e42ffc0796855, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732098249590 2024-11-20T10:24:10,958 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#C#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:10,959 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/179db8814c2e422095defb10326c372e is 50, key is test_row_0/C:col10/1732098250228/Put/seqid=0 2024-11-20T10:24:10,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741875_1051 (size=12151) 2024-11-20T10:24:10,975 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/a940aa4b46a1408d9d374a0fd8ab9b6f 2024-11-20T10:24:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/5e1791da847b478799bb31d11c002ea0 is 50, key is test_row_0/B:col10/1732098250283/Put/seqid=0 2024-11-20T10:24:11,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741876_1052 (size=12561) 2024-11-20T10:24:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:11,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:11,024 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/179db8814c2e422095defb10326c372e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/179db8814c2e422095defb10326c372e 2024-11-20T10:24:11,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741877_1053 (size=12151) 2024-11-20T10:24:11,041 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/C of 455386d186b9f64def53c239491339e4 into 179db8814c2e422095defb10326c372e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:11,042 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:11,042 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/C, priority=13, startTime=1732098250830; duration=0sec 2024-11-20T10:24:11,042 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:11,042 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:C 2024-11-20T10:24:11,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098311040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098311043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098311045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098311046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T10:24:11,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098311157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098311157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098311157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098311158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,319 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/a6a3ba3c4ab84c2493438e1010d631cd as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/a6a3ba3c4ab84c2493438e1010d631cd 2024-11-20T10:24:11,339 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/B of 455386d186b9f64def53c239491339e4 into a6a3ba3c4ab84c2493438e1010d631cd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:11,340 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:11,340 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/B, priority=13, startTime=1732098250830; duration=0sec 2024-11-20T10:24:11,340 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:11,340 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:B 2024-11-20T10:24:11,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098311368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098311368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098311368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098311368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,439 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/5e1791da847b478799bb31d11c002ea0 2024-11-20T10:24:11,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/f04794e800e74367b9b14b72f06e08cf is 50, key is test_row_0/C:col10/1732098250283/Put/seqid=0 2024-11-20T10:24:11,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741878_1054 (size=12151) 2024-11-20T10:24:11,481 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/f04794e800e74367b9b14b72f06e08cf 2024-11-20T10:24:11,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/a940aa4b46a1408d9d374a0fd8ab9b6f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/a940aa4b46a1408d9d374a0fd8ab9b6f 2024-11-20T10:24:11,510 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/a940aa4b46a1408d9d374a0fd8ab9b6f, entries=150, sequenceid=199, filesize=11.9 K 2024-11-20T10:24:11,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/5e1791da847b478799bb31d11c002ea0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/5e1791da847b478799bb31d11c002ea0 2024-11-20T10:24:11,525 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/5e1791da847b478799bb31d11c002ea0, entries=150, sequenceid=199, filesize=11.9 K 2024-11-20T10:24:11,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/f04794e800e74367b9b14b72f06e08cf as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/f04794e800e74367b9b14b72f06e08cf 2024-11-20T10:24:11,538 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/f04794e800e74367b9b14b72f06e08cf, entries=150, sequenceid=199, filesize=11.9 K 2024-11-20T10:24:11,539 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 455386d186b9f64def53c239491339e4 in 617ms, sequenceid=199, compaction requested=false 2024-11-20T10:24:11,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:11,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:11,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-20T10:24:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-20T10:24:11,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-20T10:24:11,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5680 sec 2024-11-20T10:24:11,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.5820 sec 2024-11-20T10:24:11,681 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:11,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-20T10:24:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:11,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/2f769256759d44cd9abaa19c394bf18e is 50, key is test_row_0/A:col10/1732098251679/Put/seqid=0 2024-11-20T10:24:11,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741879_1055 (size=12151) 2024-11-20T10:24:11,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098311733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098311741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098311744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098311743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098311847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098311857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098311857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098311857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:12,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098312057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:12,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098312066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098312067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:12,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098312067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:12,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/2f769256759d44cd9abaa19c394bf18e 2024-11-20T10:24:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T10:24:12,117 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:12,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-20T10:24:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T10:24:12,131 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:12,132 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:12,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:12,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/366c18f92b0e4aca823042c77335305e is 50, key is test_row_0/B:col10/1732098251679/Put/seqid=0 2024-11-20T10:24:12,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741880_1056 (size=12151) 2024-11-20T10:24:12,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/366c18f92b0e4aca823042c77335305e 2024-11-20T10:24:12,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/72bec96218494f738baeea6de1226854 is 50, key is test_row_0/C:col10/1732098251679/Put/seqid=0 2024-11-20T10:24:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T10:24:12,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T10:24:12,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:12,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:12,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:12,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:12,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:12,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741881_1057 (size=12151) 2024-11-20T10:24:12,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098312368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:12,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:12,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098312377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098312377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:12,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098312378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:12,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T10:24:12,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:12,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:12,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:12,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:12,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T10:24:12,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T10:24:12,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:12,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:12,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:12,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:12,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:12,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/72bec96218494f738baeea6de1226854 2024-11-20T10:24:12,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098312698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:12,699 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in 455386d186b9f64def53c239491339e4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:24:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:12,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/2f769256759d44cd9abaa19c394bf18e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2f769256759d44cd9abaa19c394bf18e 2024-11-20T10:24:12,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2f769256759d44cd9abaa19c394bf18e, entries=150, sequenceid=218, filesize=11.9 K 2024-11-20T10:24:12,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/366c18f92b0e4aca823042c77335305e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/366c18f92b0e4aca823042c77335305e 2024-11-20T10:24:12,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/366c18f92b0e4aca823042c77335305e, entries=150, sequenceid=218, filesize=11.9 K 2024-11-20T10:24:12,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/72bec96218494f738baeea6de1226854 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/72bec96218494f738baeea6de1226854 2024-11-20T10:24:12,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/72bec96218494f738baeea6de1226854, entries=150, sequenceid=218, filesize=11.9 K 2024-11-20T10:24:12,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T10:24:12,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:12,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:12,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:12,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:12,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 455386d186b9f64def53c239491339e4 in 1071ms, sequenceid=218, compaction requested=true 2024-11-20T10:24:12,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:12,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:12,757 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T10:24:12,760 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:12,761 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/A is initiating minor compaction (all files) 2024-11-20T10:24:12,761 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/A in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:12,761 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/f19fa2b510034e249bd76705212e939c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/a940aa4b46a1408d9d374a0fd8ab9b6f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2f769256759d44cd9abaa19c394bf18e] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=36.0 K 2024-11-20T10:24:12,763 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f19fa2b510034e249bd76705212e939c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732098249590 2024-11-20T10:24:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:12,763 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:12,763 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a940aa4b46a1408d9d374a0fd8ab9b6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732098250283 2024-11-20T10:24:12,764 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f769256759d44cd9abaa19c394bf18e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732098251042 2024-11-20T10:24:12,765 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:12,765 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/B is initiating minor compaction (all files) 2024-11-20T10:24:12,765 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/B in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:12,765 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/a6a3ba3c4ab84c2493438e1010d631cd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/5e1791da847b478799bb31d11c002ea0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/366c18f92b0e4aca823042c77335305e] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=36.0 K 2024-11-20T10:24:12,766 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a6a3ba3c4ab84c2493438e1010d631cd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732098249590 2024-11-20T10:24:12,766 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e1791da847b478799bb31d11c002ea0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732098250283 2024-11-20T10:24:12,767 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 366c18f92b0e4aca823042c77335305e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732098251042 2024-11-20T10:24:12,782 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#A#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:12,782 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/1b6f24ef1d7a4af8bc766f7b3d020aa1 is 50, key is test_row_0/A:col10/1732098251679/Put/seqid=0 2024-11-20T10:24:12,786 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#B#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:12,787 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/d58abea32932493b86feb6e3e4fd4e22 is 50, key is test_row_0/B:col10/1732098251679/Put/seqid=0 2024-11-20T10:24:12,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741882_1058 (size=12663) 2024-11-20T10:24:12,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741883_1059 (size=12663) 2024-11-20T10:24:12,824 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/d58abea32932493b86feb6e3e4fd4e22 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/d58abea32932493b86feb6e3e4fd4e22 2024-11-20T10:24:12,829 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/1b6f24ef1d7a4af8bc766f7b3d020aa1 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/1b6f24ef1d7a4af8bc766f7b3d020aa1 2024-11-20T10:24:12,834 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/B of 455386d186b9f64def53c239491339e4 into d58abea32932493b86feb6e3e4fd4e22(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:12,834 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:12,834 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/B, priority=13, startTime=1732098252763; duration=0sec 2024-11-20T10:24:12,835 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:12,835 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:B 2024-11-20T10:24:12,835 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:12,836 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:12,836 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/C is initiating minor compaction (all files) 2024-11-20T10:24:12,837 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/C in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:12,837 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/179db8814c2e422095defb10326c372e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/f04794e800e74367b9b14b72f06e08cf, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/72bec96218494f738baeea6de1226854] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=36.0 K 2024-11-20T10:24:12,838 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 179db8814c2e422095defb10326c372e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732098249590 2024-11-20T10:24:12,839 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/A of 455386d186b9f64def53c239491339e4 into 1b6f24ef1d7a4af8bc766f7b3d020aa1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:12,839 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:12,839 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/A, priority=13, startTime=1732098252756; duration=0sec 2024-11-20T10:24:12,839 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f04794e800e74367b9b14b72f06e08cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732098250283 2024-11-20T10:24:12,839 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:12,839 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:A 2024-11-20T10:24:12,840 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 72bec96218494f738baeea6de1226854, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732098251042 2024-11-20T10:24:12,862 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#C#compaction#47 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:12,863 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/9158d0592a0143d7960fd174c3db73ba is 50, key is test_row_0/C:col10/1732098251679/Put/seqid=0 2024-11-20T10:24:12,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741884_1060 (size=12663) 2024-11-20T10:24:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:12,880 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:12,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-20T10:24:12,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/8ac88ef5107746a796abd3218fb04641 is 50, key is test_row_0/A:col10/1732098251739/Put/seqid=0 2024-11-20T10:24:12,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T10:24:12,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:12,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:12,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:12,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:12,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:12,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:12,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098312912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:12,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:12,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741885_1061 (size=12151) 2024-11-20T10:24:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098312913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:12,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/8ac88ef5107746a796abd3218fb04641 2024-11-20T10:24:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:12,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098312914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:12,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:12,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098312915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:12,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/bfc03ebba6d94d158107a01e5043a32b is 50, key is test_row_0/B:col10/1732098251739/Put/seqid=0 2024-11-20T10:24:12,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741886_1062 (size=12151) 2024-11-20T10:24:13,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:13,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098313027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098313027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:13,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:13,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098313027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:13,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098313028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:13,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T10:24:13,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:13,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:13,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:13,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:13,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:13,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:13,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T10:24:13,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:13,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:13,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:13,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:13,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:13,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:13,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098313237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:13,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098313237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:13,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098313237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:13,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098313237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:13,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T10:24:13,279 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/9158d0592a0143d7960fd174c3db73ba as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9158d0592a0143d7960fd174c3db73ba 2024-11-20T10:24:13,286 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/C of 455386d186b9f64def53c239491339e4 into 9158d0592a0143d7960fd174c3db73ba(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:13,286 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:13,286 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/C, priority=13, startTime=1732098252763; duration=0sec 2024-11-20T10:24:13,286 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:13,286 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:C 2024-11-20T10:24:13,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/bfc03ebba6d94d158107a01e5043a32b 2024-11-20T10:24:13,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/1b7edb8b85274d3aa930cd457a038b84 is 50, key is test_row_0/C:col10/1732098251739/Put/seqid=0 2024-11-20T10:24:13,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741887_1063 (size=12151) 2024-11-20T10:24:13,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T10:24:13,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:13,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:13,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:13,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:13,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:13,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T10:24:13,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:13,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:13,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:13,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:13,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:13,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098313547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:13,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:13,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098313547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:13,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:13,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098313547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:13,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098313547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:13,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T10:24:13,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:13,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:13,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:13,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:13,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:13,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/1b7edb8b85274d3aa930cd457a038b84 2024-11-20T10:24:13,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/8ac88ef5107746a796abd3218fb04641 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/8ac88ef5107746a796abd3218fb04641 2024-11-20T10:24:13,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/8ac88ef5107746a796abd3218fb04641, entries=150, sequenceid=242, filesize=11.9 K 2024-11-20T10:24:13,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/bfc03ebba6d94d158107a01e5043a32b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/bfc03ebba6d94d158107a01e5043a32b 2024-11-20T10:24:13,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/bfc03ebba6d94d158107a01e5043a32b, entries=150, sequenceid=242, filesize=11.9 K 2024-11-20T10:24:13,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/1b7edb8b85274d3aa930cd457a038b84 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1b7edb8b85274d3aa930cd457a038b84 2024-11-20T10:24:13,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1b7edb8b85274d3aa930cd457a038b84, entries=150, sequenceid=242, filesize=11.9 K 2024-11-20T10:24:13,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 455386d186b9f64def53c239491339e4 in 914ms, sequenceid=242, compaction requested=false 2024-11-20T10:24:13,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:13,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T10:24:13,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:13,830 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-20T10:24:13,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/512e66c4af2e449a9775ee181f3d748b is 50, key is test_row_0/A:col10/1732098252913/Put/seqid=0 2024-11-20T10:24:13,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741888_1064 (size=12151) 2024-11-20T10:24:14,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:14,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098314092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098314090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098314096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098314097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098314206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098314206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098314206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098314207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,244 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/512e66c4af2e449a9775ee181f3d748b 2024-11-20T10:24:14,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/0ca747363f654159ad4664275325bffb is 50, key is test_row_0/B:col10/1732098252913/Put/seqid=0 2024-11-20T10:24:14,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741889_1065 (size=12151) 2024-11-20T10:24:14,262 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/0ca747363f654159ad4664275325bffb 2024-11-20T10:24:14,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/9b5b13a52f264076a3e33eb2d869b370 is 50, key is test_row_0/C:col10/1732098252913/Put/seqid=0 2024-11-20T10:24:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T10:24:14,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741890_1066 (size=12151) 2024-11-20T10:24:14,285 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/9b5b13a52f264076a3e33eb2d869b370 2024-11-20T10:24:14,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/512e66c4af2e449a9775ee181f3d748b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/512e66c4af2e449a9775ee181f3d748b 2024-11-20T10:24:14,298 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/512e66c4af2e449a9775ee181f3d748b, entries=150, sequenceid=258, filesize=11.9 K 2024-11-20T10:24:14,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/0ca747363f654159ad4664275325bffb as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/0ca747363f654159ad4664275325bffb 2024-11-20T10:24:14,308 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/0ca747363f654159ad4664275325bffb, entries=150, sequenceid=258, filesize=11.9 K 2024-11-20T10:24:14,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/9b5b13a52f264076a3e33eb2d869b370 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9b5b13a52f264076a3e33eb2d869b370 2024-11-20T10:24:14,321 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9b5b13a52f264076a3e33eb2d869b370, entries=150, sequenceid=258, filesize=11.9 K 2024-11-20T10:24:14,322 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 455386d186b9f64def53c239491339e4 in 492ms, sequenceid=258, compaction requested=true 2024-11-20T10:24:14,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:14,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:14,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-20T10:24:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-20T10:24:14,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-20T10:24:14,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1950 sec 2024-11-20T10:24:14,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.2070 sec 2024-11-20T10:24:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:14,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T10:24:14,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/bad639157b8f443eb4475bf2f6b2d036 is 50, key is test_row_0/A:col10/1732098254421/Put/seqid=0 2024-11-20T10:24:14,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098314438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741891_1067 (size=12301) 2024-11-20T10:24:14,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098314438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098314443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098314445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/bad639157b8f443eb4475bf2f6b2d036 2024-11-20T10:24:14,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/dc76c05f97ef4b4e9210f582e308217b is 50, key is test_row_0/B:col10/1732098254421/Put/seqid=0 2024-11-20T10:24:14,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741892_1068 (size=12301) 2024-11-20T10:24:14,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098314556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098314556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098314557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098314557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098314767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098314767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098314767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098314776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:14,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:14,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:14,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/dc76c05f97ef4b4e9210f582e308217b 2024-11-20T10:24:14,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/4d7cd14052b2455d9ffb035cc5fd1626 is 50, key is test_row_0/C:col10/1732098254421/Put/seqid=0 2024-11-20T10:24:14,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741893_1069 (size=12301) 2024-11-20T10:24:15,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098315077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098315077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098315077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:15,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098315086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:15,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/4d7cd14052b2455d9ffb035cc5fd1626 2024-11-20T10:24:15,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/bad639157b8f443eb4475bf2f6b2d036 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bad639157b8f443eb4475bf2f6b2d036 2024-11-20T10:24:15,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bad639157b8f443eb4475bf2f6b2d036, entries=150, sequenceid=283, filesize=12.0 K 2024-11-20T10:24:15,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/dc76c05f97ef4b4e9210f582e308217b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/dc76c05f97ef4b4e9210f582e308217b 2024-11-20T10:24:15,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/dc76c05f97ef4b4e9210f582e308217b, entries=150, sequenceid=283, filesize=12.0 K 2024-11-20T10:24:15,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/4d7cd14052b2455d9ffb035cc5fd1626 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/4d7cd14052b2455d9ffb035cc5fd1626 2024-11-20T10:24:15,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/4d7cd14052b2455d9ffb035cc5fd1626, entries=150, sequenceid=283, filesize=12.0 K 2024-11-20T10:24:15,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 455386d186b9f64def53c239491339e4 in 890ms, sequenceid=283, compaction requested=true 2024-11-20T10:24:15,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:15,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:15,312 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:24:15,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:15,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:15,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:15,312 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:24:15,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:15,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:15,314 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:24:15,314 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:24:15,314 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/B is initiating minor compaction (all files) 2024-11-20T10:24:15,314 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/A is initiating minor compaction (all files) 2024-11-20T10:24:15,314 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/B in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:15,314 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/A in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:15,314 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/1b6f24ef1d7a4af8bc766f7b3d020aa1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/8ac88ef5107746a796abd3218fb04641, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/512e66c4af2e449a9775ee181f3d748b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bad639157b8f443eb4475bf2f6b2d036] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=48.1 K 2024-11-20T10:24:15,314 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/d58abea32932493b86feb6e3e4fd4e22, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/bfc03ebba6d94d158107a01e5043a32b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/0ca747363f654159ad4664275325bffb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/dc76c05f97ef4b4e9210f582e308217b] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=48.1 K 2024-11-20T10:24:15,314 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b6f24ef1d7a4af8bc766f7b3d020aa1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732098251042 2024-11-20T10:24:15,314 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d58abea32932493b86feb6e3e4fd4e22, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732098251042 2024-11-20T10:24:15,315 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ac88ef5107746a796abd3218fb04641, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732098251736 2024-11-20T10:24:15,315 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting bfc03ebba6d94d158107a01e5043a32b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732098251736 2024-11-20T10:24:15,315 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 512e66c4af2e449a9775ee181f3d748b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732098252908 2024-11-20T10:24:15,315 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ca747363f654159ad4664275325bffb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732098252908 2024-11-20T10:24:15,315 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting bad639157b8f443eb4475bf2f6b2d036, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732098254094 2024-11-20T10:24:15,316 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting dc76c05f97ef4b4e9210f582e308217b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732098254094 2024-11-20T10:24:15,331 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:15,332 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/039e49b2da8748eba57b08e10f82d441 is 50, key is test_row_0/A:col10/1732098254421/Put/seqid=0 2024-11-20T10:24:15,336 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#B#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:15,337 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/54d62cfdc629476c823abe710ad0b7fa is 50, key is test_row_0/B:col10/1732098254421/Put/seqid=0 2024-11-20T10:24:15,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741894_1070 (size=12949) 2024-11-20T10:24:15,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741895_1071 (size=12949) 2024-11-20T10:24:15,361 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/039e49b2da8748eba57b08e10f82d441 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/039e49b2da8748eba57b08e10f82d441 2024-11-20T10:24:15,369 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/54d62cfdc629476c823abe710ad0b7fa as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/54d62cfdc629476c823abe710ad0b7fa 2024-11-20T10:24:15,373 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 455386d186b9f64def53c239491339e4/A of 455386d186b9f64def53c239491339e4 into 039e49b2da8748eba57b08e10f82d441(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:15,373 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:15,373 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/A, priority=12, startTime=1732098255312; duration=0sec 2024-11-20T10:24:15,376 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:15,376 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:A 2024-11-20T10:24:15,376 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:24:15,378 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 455386d186b9f64def53c239491339e4/B of 455386d186b9f64def53c239491339e4 into 54d62cfdc629476c823abe710ad0b7fa(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:15,378 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:15,379 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/B, priority=12, startTime=1732098255312; duration=0sec 2024-11-20T10:24:15,379 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:15,379 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:B 2024-11-20T10:24:15,381 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:24:15,382 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/C is initiating minor compaction (all files) 2024-11-20T10:24:15,382 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/C in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:15,383 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9158d0592a0143d7960fd174c3db73ba, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1b7edb8b85274d3aa930cd457a038b84, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9b5b13a52f264076a3e33eb2d869b370, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/4d7cd14052b2455d9ffb035cc5fd1626] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=48.1 K 2024-11-20T10:24:15,384 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9158d0592a0143d7960fd174c3db73ba, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732098251042 2024-11-20T10:24:15,386 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b7edb8b85274d3aa930cd457a038b84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732098251736 2024-11-20T10:24:15,386 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b5b13a52f264076a3e33eb2d869b370, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732098252908 2024-11-20T10:24:15,387 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d7cd14052b2455d9ffb035cc5fd1626, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732098254094 2024-11-20T10:24:15,401 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:15,402 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/cc689d92e95f438f8c94b7cae42d30e1 is 50, key is test_row_0/C:col10/1732098254421/Put/seqid=0 2024-11-20T10:24:15,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741896_1072 (size=12949) 2024-11-20T10:24:15,424 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/cc689d92e95f438f8c94b7cae42d30e1 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/cc689d92e95f438f8c94b7cae42d30e1 2024-11-20T10:24:15,433 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 455386d186b9f64def53c239491339e4/C of 455386d186b9f64def53c239491339e4 into cc689d92e95f438f8c94b7cae42d30e1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:15,433 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:15,433 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/C, priority=12, startTime=1732098255312; duration=0sec 2024-11-20T10:24:15,433 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:15,433 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:C 2024-11-20T10:24:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:15,589 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:15,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-20T10:24:15,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/78bf114bf0614253bae2f9940f8d7683 is 50, key is test_row_0/A:col10/1732098254440/Put/seqid=0 2024-11-20T10:24:15,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741897_1073 (size=14741) 2024-11-20T10:24:15,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/78bf114bf0614253bae2f9940f8d7683 2024-11-20T10:24:15,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/360b57b089de4b538f9dbf77dce891b2 is 50, key is test_row_0/B:col10/1732098254440/Put/seqid=0 2024-11-20T10:24:15,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741898_1074 (size=12301) 2024-11-20T10:24:15,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098315628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098315629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:15,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098315634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098315635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:15,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098315746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098315747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098315747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098315747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:15,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098315957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098315957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098315957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:15,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098315966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:16,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/360b57b089de4b538f9dbf77dce891b2 2024-11-20T10:24:16,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/7ac0785b771947048afdfa6617a307a8 is 50, key is test_row_0/C:col10/1732098254440/Put/seqid=0 2024-11-20T10:24:16,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741899_1075 (size=12301) 2024-11-20T10:24:16,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098316267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098316267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098316267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:16,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098316277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T10:24:16,298 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:16,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-20T10:24:16,303 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T10:24:16,304 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:16,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T10:24:16,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/7ac0785b771947048afdfa6617a307a8 2024-11-20T10:24:16,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T10:24:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:16,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:16,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:16,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/78bf114bf0614253bae2f9940f8d7683 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/78bf114bf0614253bae2f9940f8d7683 2024-11-20T10:24:16,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/78bf114bf0614253bae2f9940f8d7683, entries=200, sequenceid=301, filesize=14.4 K 2024-11-20T10:24:16,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/360b57b089de4b538f9dbf77dce891b2 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/360b57b089de4b538f9dbf77dce891b2 2024-11-20T10:24:16,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/360b57b089de4b538f9dbf77dce891b2, entries=150, sequenceid=301, filesize=12.0 K 2024-11-20T10:24:16,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/7ac0785b771947048afdfa6617a307a8 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/7ac0785b771947048afdfa6617a307a8 2024-11-20T10:24:16,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/7ac0785b771947048afdfa6617a307a8, entries=150, sequenceid=301, filesize=12.0 K 2024-11-20T10:24:16,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 455386d186b9f64def53c239491339e4 in 904ms, sequenceid=301, compaction requested=false 2024-11-20T10:24:16,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:16,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T10:24:16,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:16,616 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-20T10:24:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T10:24:16,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/27cfc780c1d4410eb5bebf5baf2e3416 is 50, key is test_row_0/A:col10/1732098255631/Put/seqid=0 2024-11-20T10:24:16,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741900_1076 (size=12301) 2024-11-20T10:24:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:16,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:16,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098316783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:16,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098316783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:16,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098316785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098316785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:16,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098316786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:16,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:16,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098316896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098316896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:16,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:16,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098316897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098316896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T10:24:17,067 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/27cfc780c1d4410eb5bebf5baf2e3416 2024-11-20T10:24:17,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/3919e4ac689046de9da21a9ae29f6506 is 50, key is test_row_0/B:col10/1732098255631/Put/seqid=0 2024-11-20T10:24:17,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741901_1077 (size=12301) 2024-11-20T10:24:17,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098317106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098317107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:17,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098317107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:17,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098317107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:17,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098317416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:17,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098317417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098317417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098317417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:17,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T10:24:17,480 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/3919e4ac689046de9da21a9ae29f6506 2024-11-20T10:24:17,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/6ac41fa359714d769d8733639c754510 is 50, key is test_row_0/C:col10/1732098255631/Put/seqid=0 2024-11-20T10:24:17,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741902_1078 (size=12301) 2024-11-20T10:24:17,499 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/6ac41fa359714d769d8733639c754510 2024-11-20T10:24:17,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/27cfc780c1d4410eb5bebf5baf2e3416 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/27cfc780c1d4410eb5bebf5baf2e3416 2024-11-20T10:24:17,512 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/27cfc780c1d4410eb5bebf5baf2e3416, entries=150, sequenceid=322, filesize=12.0 K 2024-11-20T10:24:17,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/3919e4ac689046de9da21a9ae29f6506 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/3919e4ac689046de9da21a9ae29f6506 2024-11-20T10:24:17,519 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/3919e4ac689046de9da21a9ae29f6506, entries=150, sequenceid=322, filesize=12.0 K 2024-11-20T10:24:17,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/6ac41fa359714d769d8733639c754510 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/6ac41fa359714d769d8733639c754510 2024-11-20T10:24:17,530 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/6ac41fa359714d769d8733639c754510, entries=150, sequenceid=322, filesize=12.0 K 2024-11-20T10:24:17,531 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 455386d186b9f64def53c239491339e4 in 916ms, sequenceid=322, compaction requested=true 2024-11-20T10:24:17,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:17,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:17,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-20T10:24:17,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-20T10:24:17,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T10:24:17,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2290 sec 2024-11-20T10:24:17,538 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2370 sec 2024-11-20T10:24:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:17,800 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:17,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T10:24:17,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/2d43716fefb443e58cd108369600a76d is 50, key is test_row_0/A:col10/1732098256782/Put/seqid=0 2024-11-20T10:24:17,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741903_1079 (size=12301) 2024-11-20T10:24:17,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098317863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:17,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098317926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:17,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098317927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:17,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098317927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:17,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098317937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:17,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098317976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:18,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098318186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:18,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/2d43716fefb443e58cd108369600a76d 2024-11-20T10:24:18,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/420c921d21a14b20b20c64eab7ca6575 is 50, key is test_row_0/B:col10/1732098256782/Put/seqid=0 2024-11-20T10:24:18,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741904_1080 (size=12301) 2024-11-20T10:24:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T10:24:18,447 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:18,447 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-20T10:24:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T10:24:18,449 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:18,450 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:18,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:18,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098318497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:18,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:18,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:18,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T10:24:18,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T10:24:18,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:18,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:18,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:18,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:18,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:18,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/420c921d21a14b20b20c64eab7ca6575 2024-11-20T10:24:18,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/c928fea9922b428eac1fd5b8a7afe1d0 is 50, key is test_row_0/C:col10/1732098256782/Put/seqid=0 2024-11-20T10:24:18,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741905_1081 (size=12301) 2024-11-20T10:24:18,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/c928fea9922b428eac1fd5b8a7afe1d0 2024-11-20T10:24:18,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/2d43716fefb443e58cd108369600a76d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2d43716fefb443e58cd108369600a76d 2024-11-20T10:24:18,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2d43716fefb443e58cd108369600a76d, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T10:24:18,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/420c921d21a14b20b20c64eab7ca6575 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/420c921d21a14b20b20c64eab7ca6575 2024-11-20T10:24:18,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/420c921d21a14b20b20c64eab7ca6575, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T10:24:18,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/c928fea9922b428eac1fd5b8a7afe1d0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/c928fea9922b428eac1fd5b8a7afe1d0 2024-11-20T10:24:18,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/c928fea9922b428eac1fd5b8a7afe1d0, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T10:24:18,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 455386d186b9f64def53c239491339e4 in 925ms, sequenceid=339, compaction requested=true 2024-11-20T10:24:18,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:18,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:18,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:18,725 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:24:18,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:18,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:18,725 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:24:18,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:18,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:18,727 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:24:18,727 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:24:18,727 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/B is initiating minor compaction (all files) 2024-11-20T10:24:18,727 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/A is initiating minor compaction (all files) 2024-11-20T10:24:18,727 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/B in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:18,727 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/A in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:18,727 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/039e49b2da8748eba57b08e10f82d441, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/78bf114bf0614253bae2f9940f8d7683, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/27cfc780c1d4410eb5bebf5baf2e3416, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2d43716fefb443e58cd108369600a76d] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=51.1 K 2024-11-20T10:24:18,727 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/54d62cfdc629476c823abe710ad0b7fa, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/360b57b089de4b538f9dbf77dce891b2, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/3919e4ac689046de9da21a9ae29f6506, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/420c921d21a14b20b20c64eab7ca6575] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=48.7 K 2024-11-20T10:24:18,728 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 039e49b2da8748eba57b08e10f82d441, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732098254094 2024-11-20T10:24:18,728 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 54d62cfdc629476c823abe710ad0b7fa, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732098254094 2024-11-20T10:24:18,728 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 360b57b089de4b538f9dbf77dce891b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732098254440 2024-11-20T10:24:18,728 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78bf114bf0614253bae2f9940f8d7683, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732098254440 2024-11-20T10:24:18,728 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3919e4ac689046de9da21a9ae29f6506, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732098255612 2024-11-20T10:24:18,729 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27cfc780c1d4410eb5bebf5baf2e3416, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732098255612 2024-11-20T10:24:18,729 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 420c921d21a14b20b20c64eab7ca6575, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732098256770 2024-11-20T10:24:18,730 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d43716fefb443e58cd108369600a76d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732098256770 2024-11-20T10:24:18,750 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#B#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:18,751 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/8553ac7ae75640549f7123c1dd263043 is 50, key is test_row_0/B:col10/1732098256782/Put/seqid=0 2024-11-20T10:24:18,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T10:24:18,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:18,756 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T10:24:18,762 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#A#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:18,762 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/d3c549a0b7684172b91261bd8edc6ab5 is 50, key is test_row_0/A:col10/1732098256782/Put/seqid=0 2024-11-20T10:24:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T10:24:18,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/77405886c8f74e5e9d78603f6559348d is 50, key is test_row_0/A:col10/1732098257846/Put/seqid=0 2024-11-20T10:24:18,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741906_1082 (size=13085) 2024-11-20T10:24:18,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741907_1083 (size=13085) 2024-11-20T10:24:18,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741908_1084 (size=12301) 2024-11-20T10:24:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:18,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:18,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098318966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:18,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098318966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:18,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098318972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:18,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098318973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:19,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098319006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T10:24:19,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098319077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098319077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:19,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098319086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098319087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:19,223 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/8553ac7ae75640549f7123c1dd263043 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/8553ac7ae75640549f7123c1dd263043 2024-11-20T10:24:19,235 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 455386d186b9f64def53c239491339e4/B of 455386d186b9f64def53c239491339e4 into 8553ac7ae75640549f7123c1dd263043(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:19,235 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:19,235 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/B, priority=12, startTime=1732098258725; duration=0sec 2024-11-20T10:24:19,235 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/77405886c8f74e5e9d78603f6559348d 2024-11-20T10:24:19,235 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:19,236 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:B 2024-11-20T10:24:19,236 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:24:19,246 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:24:19,247 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/C is initiating minor compaction (all files) 2024-11-20T10:24:19,247 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/C in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:19,247 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/cc689d92e95f438f8c94b7cae42d30e1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/7ac0785b771947048afdfa6617a307a8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/6ac41fa359714d769d8733639c754510, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/c928fea9922b428eac1fd5b8a7afe1d0] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=48.7 K 2024-11-20T10:24:19,253 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting cc689d92e95f438f8c94b7cae42d30e1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732098254094 2024-11-20T10:24:19,254 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ac0785b771947048afdfa6617a307a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732098254440 2024-11-20T10:24:19,255 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/d3c549a0b7684172b91261bd8edc6ab5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/d3c549a0b7684172b91261bd8edc6ab5 2024-11-20T10:24:19,256 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ac41fa359714d769d8733639c754510, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732098255612 2024-11-20T10:24:19,258 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c928fea9922b428eac1fd5b8a7afe1d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732098256770 2024-11-20T10:24:19,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/c0d477de169f48ae8598623f1743b7f3 is 50, key is test_row_0/B:col10/1732098257846/Put/seqid=0 2024-11-20T10:24:19,271 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 455386d186b9f64def53c239491339e4/A of 455386d186b9f64def53c239491339e4 into d3c549a0b7684172b91261bd8edc6ab5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:19,271 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:19,271 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/A, priority=12, startTime=1732098258725; duration=0sec 2024-11-20T10:24:19,272 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:19,272 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:A 2024-11-20T10:24:19,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741909_1085 (size=12301) 2024-11-20T10:24:19,280 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#C#compaction#73 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:19,280 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/1dc1a78941a449acb245d23444d3500b is 50, key is test_row_0/C:col10/1732098256782/Put/seqid=0 2024-11-20T10:24:19,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098319287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:19,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098319287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:19,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741910_1086 (size=13085) 2024-11-20T10:24:19,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098319296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:19,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098319297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T10:24:19,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098319597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:19,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:19,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098319599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:19,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098319606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:19,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098319607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:19,679 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/c0d477de169f48ae8598623f1743b7f3 2024-11-20T10:24:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/3de6b1c0b7b04589a255dab778d8b54a is 50, key is test_row_0/C:col10/1732098257846/Put/seqid=0 2024-11-20T10:24:19,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741911_1087 (size=12301) 2024-11-20T10:24:19,699 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/1dc1a78941a449acb245d23444d3500b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1dc1a78941a449acb245d23444d3500b 2024-11-20T10:24:19,706 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 455386d186b9f64def53c239491339e4/C of 455386d186b9f64def53c239491339e4 into 1dc1a78941a449acb245d23444d3500b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:19,706 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:19,706 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/C, priority=12, startTime=1732098258725; duration=0sec 2024-11-20T10:24:19,706 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:19,706 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:C 2024-11-20T10:24:20,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098320026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:20,034 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_1' in 455386d186b9f64def53c239491339e4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:24:20,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:20,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:20,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:20,091 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/3de6b1c0b7b04589a255dab778d8b54a 2024-11-20T10:24:20,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/77405886c8f74e5e9d78603f6559348d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/77405886c8f74e5e9d78603f6559348d 2024-11-20T10:24:20,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:20,110 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/77405886c8f74e5e9d78603f6559348d, entries=150, sequenceid=358, filesize=12.0 K 2024-11-20T10:24:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098320106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:20,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098320106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:20,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/c0d477de169f48ae8598623f1743b7f3 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c0d477de169f48ae8598623f1743b7f3 2024-11-20T10:24:20,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:20,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098320117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098320117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:20,122 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c0d477de169f48ae8598623f1743b7f3, entries=150, sequenceid=358, filesize=12.0 K 2024-11-20T10:24:20,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:20,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:20,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:20,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/3de6b1c0b7b04589a255dab778d8b54a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/3de6b1c0b7b04589a255dab778d8b54a 2024-11-20T10:24:20,133 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/3de6b1c0b7b04589a255dab778d8b54a, entries=150, sequenceid=358, filesize=12.0 K 2024-11-20T10:24:20,134 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 455386d186b9f64def53c239491339e4 in 1378ms, sequenceid=358, compaction requested=false 2024-11-20T10:24:20,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:20,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:20,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-20T10:24:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-20T10:24:20,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-20T10:24:20,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6860 sec 2024-11-20T10:24:20,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.6920 sec 2024-11-20T10:24:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T10:24:20,597 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:20,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-20T10:24:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T10:24:20,602 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:20,603 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:20,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T10:24:20,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T10:24:20,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:20,756 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T10:24:20,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/3cbfbfacdb4347ad838a6802ab4df067 is 50, key is test_row_0/A:col10/1732098258971/Put/seqid=0 2024-11-20T10:24:20,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741912_1088 (size=12301) 2024-11-20T10:24:20,769 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/3cbfbfacdb4347ad838a6802ab4df067 2024-11-20T10:24:20,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/a28a6ff5656e462795a619f1f53fffa4 is 50, key is test_row_0/B:col10/1732098258971/Put/seqid=0 2024-11-20T10:24:20,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741913_1089 (size=12301) 2024-11-20T10:24:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T10:24:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:21,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:21,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098321143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098321143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098321144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098321145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,196 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/a28a6ff5656e462795a619f1f53fffa4 2024-11-20T10:24:21,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/cc7d092ed1254676b404e203ddfedf07 is 50, key is test_row_0/C:col10/1732098258971/Put/seqid=0 2024-11-20T10:24:21,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741914_1090 (size=12301) 2024-11-20T10:24:21,210 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/cc7d092ed1254676b404e203ddfedf07 2024-11-20T10:24:21,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/3cbfbfacdb4347ad838a6802ab4df067 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/3cbfbfacdb4347ad838a6802ab4df067 2024-11-20T10:24:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T10:24:21,229 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/3cbfbfacdb4347ad838a6802ab4df067, entries=150, sequenceid=379, filesize=12.0 K 2024-11-20T10:24:21,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/a28a6ff5656e462795a619f1f53fffa4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/a28a6ff5656e462795a619f1f53fffa4 2024-11-20T10:24:21,241 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/a28a6ff5656e462795a619f1f53fffa4, entries=150, sequenceid=379, filesize=12.0 K 2024-11-20T10:24:21,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/cc7d092ed1254676b404e203ddfedf07 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/cc7d092ed1254676b404e203ddfedf07 2024-11-20T10:24:21,253 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/cc7d092ed1254676b404e203ddfedf07, entries=150, sequenceid=379, filesize=12.0 K 2024-11-20T10:24:21,254 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 455386d186b9f64def53c239491339e4 in 498ms, sequenceid=379, compaction requested=true 2024-11-20T10:24:21,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:21,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:21,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-20T10:24:21,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-20T10:24:21,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-20T10:24:21,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 652 msec 2024-11-20T10:24:21,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:21,260 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:21,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T10:24:21,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 663 msec 2024-11-20T10:24:21,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/993dea8c9f364a75923af4974eae647a is 50, key is test_row_0/A:col10/1732098261259/Put/seqid=0 2024-11-20T10:24:21,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741915_1091 (size=12301) 2024-11-20T10:24:21,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/993dea8c9f364a75923af4974eae647a 2024-11-20T10:24:21,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/9417e20f543e4894930ab60c99cd481e is 50, key is test_row_0/B:col10/1732098261259/Put/seqid=0 2024-11-20T10:24:21,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741916_1092 (size=12301) 2024-11-20T10:24:21,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/9417e20f543e4894930ab60c99cd481e 2024-11-20T10:24:21,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/a939cae2cddd4e78b00f7409321663ce is 50, key is test_row_0/C:col10/1732098261259/Put/seqid=0 2024-11-20T10:24:21,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098321283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741917_1093 (size=12301) 2024-11-20T10:24:21,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098321318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098321318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098321318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098321426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098321427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098321427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098321427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098321636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098321636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098321637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098321637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/a939cae2cddd4e78b00f7409321663ce 2024-11-20T10:24:21,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/993dea8c9f364a75923af4974eae647a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/993dea8c9f364a75923af4974eae647a 2024-11-20T10:24:21,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/993dea8c9f364a75923af4974eae647a, entries=150, sequenceid=399, filesize=12.0 K 2024-11-20T10:24:21,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/9417e20f543e4894930ab60c99cd481e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/9417e20f543e4894930ab60c99cd481e 2024-11-20T10:24:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T10:24:21,737 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:21,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-20T10:24:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T10:24:21,740 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:21,741 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:21,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:21,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/9417e20f543e4894930ab60c99cd481e, entries=150, sequenceid=399, filesize=12.0 K 2024-11-20T10:24:21,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/a939cae2cddd4e78b00f7409321663ce as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/a939cae2cddd4e78b00f7409321663ce 2024-11-20T10:24:21,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/a939cae2cddd4e78b00f7409321663ce, entries=150, sequenceid=399, filesize=12.0 K 2024-11-20T10:24:21,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 455386d186b9f64def53c239491339e4 in 491ms, sequenceid=399, compaction requested=true 2024-11-20T10:24:21,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:21,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:21,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:21,751 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:24:21,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:21,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:21,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:21,752 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:24:21,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:21,753 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:24:21,753 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:24:21,753 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/B is initiating minor compaction (all files) 2024-11-20T10:24:21,753 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/A is initiating minor compaction (all files) 2024-11-20T10:24:21,753 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/B in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:21,753 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/A in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:21,753 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/d3c549a0b7684172b91261bd8edc6ab5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/77405886c8f74e5e9d78603f6559348d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/3cbfbfacdb4347ad838a6802ab4df067, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/993dea8c9f364a75923af4974eae647a] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=48.8 K 2024-11-20T10:24:21,753 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/8553ac7ae75640549f7123c1dd263043, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c0d477de169f48ae8598623f1743b7f3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/a28a6ff5656e462795a619f1f53fffa4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/9417e20f543e4894930ab60c99cd481e] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=48.8 K 2024-11-20T10:24:21,754 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8553ac7ae75640549f7123c1dd263043, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732098256770 2024-11-20T10:24:21,754 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3c549a0b7684172b91261bd8edc6ab5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732098256770 2024-11-20T10:24:21,754 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c0d477de169f48ae8598623f1743b7f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732098257846 2024-11-20T10:24:21,754 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77405886c8f74e5e9d78603f6559348d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732098257846 2024-11-20T10:24:21,755 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a28a6ff5656e462795a619f1f53fffa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732098258964 2024-11-20T10:24:21,755 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cbfbfacdb4347ad838a6802ab4df067, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732098258964 2024-11-20T10:24:21,755 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9417e20f543e4894930ab60c99cd481e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732098261144 2024-11-20T10:24:21,755 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 993dea8c9f364a75923af4974eae647a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732098261144 2024-11-20T10:24:21,769 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#B#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:21,769 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/3879fd93cdae4b5cbdce541b529a5817 is 50, key is test_row_0/B:col10/1732098261259/Put/seqid=0 2024-11-20T10:24:21,775 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#A#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:21,775 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/0cecbfb8e710465c8eb0500245a5ea85 is 50, key is test_row_0/A:col10/1732098261259/Put/seqid=0 2024-11-20T10:24:21,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741918_1094 (size=13221) 2024-11-20T10:24:21,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741919_1095 (size=13221) 2024-11-20T10:24:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T10:24:21,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T10:24:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:21,893 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T10:24:21,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/7eada38c94fc402487b634753c320858 is 50, key is test_row_0/A:col10/1732098261282/Put/seqid=0 2024-11-20T10:24:21,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741920_1096 (size=12301) 2024-11-20T10:24:21,906 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/7eada38c94fc402487b634753c320858 2024-11-20T10:24:21,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/d4b77137d2974d068dceb53805d9472c is 50, key is test_row_0/B:col10/1732098261282/Put/seqid=0 2024-11-20T10:24:21,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741921_1097 (size=12301) 2024-11-20T10:24:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:21,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:21,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098321966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098321968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098321972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098321972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:21,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:21,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T10:24:22,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098322057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,060 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_1' in 455386d186b9f64def53c239491339e4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:24:22,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098322076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098322077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098322087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098322087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,185 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/3879fd93cdae4b5cbdce541b529a5817 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/3879fd93cdae4b5cbdce541b529a5817 2024-11-20T10:24:22,193 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 455386d186b9f64def53c239491339e4/B of 455386d186b9f64def53c239491339e4 into 3879fd93cdae4b5cbdce541b529a5817(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:22,193 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:22,193 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/B, priority=12, startTime=1732098261751; duration=0sec 2024-11-20T10:24:22,194 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:22,194 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:B 2024-11-20T10:24:22,194 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:24:22,194 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/0cecbfb8e710465c8eb0500245a5ea85 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/0cecbfb8e710465c8eb0500245a5ea85 2024-11-20T10:24:22,195 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:24:22,195 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/C is initiating minor compaction (all files) 2024-11-20T10:24:22,196 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/C in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:22,196 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1dc1a78941a449acb245d23444d3500b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/3de6b1c0b7b04589a255dab778d8b54a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/cc7d092ed1254676b404e203ddfedf07, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/a939cae2cddd4e78b00f7409321663ce] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=48.8 K 2024-11-20T10:24:22,196 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dc1a78941a449acb245d23444d3500b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732098256770 2024-11-20T10:24:22,197 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3de6b1c0b7b04589a255dab778d8b54a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732098257846 2024-11-20T10:24:22,198 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting cc7d092ed1254676b404e203ddfedf07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732098258964 2024-11-20T10:24:22,200 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a939cae2cddd4e78b00f7409321663ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732098261144 2024-11-20T10:24:22,201 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 455386d186b9f64def53c239491339e4/A of 455386d186b9f64def53c239491339e4 into 0cecbfb8e710465c8eb0500245a5ea85(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:22,201 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:22,201 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/A, priority=12, startTime=1732098261751; duration=0sec 2024-11-20T10:24:22,202 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:22,202 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:A 2024-11-20T10:24:22,211 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#C#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:22,212 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/8fcf61f174884c7a863590529bbd30d3 is 50, key is test_row_0/C:col10/1732098261259/Put/seqid=0 2024-11-20T10:24:22,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741922_1098 (size=13221) 2024-11-20T10:24:22,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098322287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098322287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098322306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098322306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,326 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/d4b77137d2974d068dceb53805d9472c 2024-11-20T10:24:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/931847685d114d438bd08635c8cbd234 is 50, key is test_row_0/C:col10/1732098261282/Put/seqid=0 2024-11-20T10:24:22,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741923_1099 (size=12301) 2024-11-20T10:24:22,342 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/931847685d114d438bd08635c8cbd234 2024-11-20T10:24:22,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/7eada38c94fc402487b634753c320858 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/7eada38c94fc402487b634753c320858 2024-11-20T10:24:22,352 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/7eada38c94fc402487b634753c320858, entries=150, sequenceid=416, filesize=12.0 K 2024-11-20T10:24:22,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/d4b77137d2974d068dceb53805d9472c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/d4b77137d2974d068dceb53805d9472c 2024-11-20T10:24:22,362 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/d4b77137d2974d068dceb53805d9472c, entries=150, sequenceid=416, filesize=12.0 K 2024-11-20T10:24:22,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/931847685d114d438bd08635c8cbd234 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/931847685d114d438bd08635c8cbd234 2024-11-20T10:24:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T10:24:22,371 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/931847685d114d438bd08635c8cbd234, entries=150, sequenceid=416, filesize=12.0 K 2024-11-20T10:24:22,372 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 455386d186b9f64def53c239491339e4 in 479ms, sequenceid=416, compaction requested=false 2024-11-20T10:24:22,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:22,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:22,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-20T10:24:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-20T10:24:22,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-20T10:24:22,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 633 msec 2024-11-20T10:24:22,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 640 msec 2024-11-20T10:24:22,599 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:22,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T10:24:22,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/678dfa75cc1647249430d445ff48a13b is 50, key is test_row_0/A:col10/1732098261970/Put/seqid=0 2024-11-20T10:24:22,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741924_1100 (size=14741) 2024-11-20T10:24:22,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/678dfa75cc1647249430d445ff48a13b 2024-11-20T10:24:22,629 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/8fcf61f174884c7a863590529bbd30d3 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/8fcf61f174884c7a863590529bbd30d3 2024-11-20T10:24:22,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098322626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098322629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098322631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098322633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,640 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 455386d186b9f64def53c239491339e4/C of 455386d186b9f64def53c239491339e4 into 8fcf61f174884c7a863590529bbd30d3(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:22,640 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:22,640 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/C, priority=12, startTime=1732098261751; duration=0sec 2024-11-20T10:24:22,640 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:22,640 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:C 2024-11-20T10:24:22,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/f01b1c8a9c8b4583bc41c0bc0879c0a6 is 50, key is test_row_0/B:col10/1732098261970/Put/seqid=0 2024-11-20T10:24:22,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741925_1101 (size=12301) 2024-11-20T10:24:22,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098322736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098322746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098322747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098322747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T10:24:22,877 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:22,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-20T10:24:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T10:24:22,880 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:22,880 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:22,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:22,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098322947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098322957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098322957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098322957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T10:24:23,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T10:24:23,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:23,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:23,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/f01b1c8a9c8b4583bc41c0bc0879c0a6 2024-11-20T10:24:23,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/41ec57ad6d03442cbeeb873646485eaf is 50, key is test_row_0/C:col10/1732098261970/Put/seqid=0 2024-11-20T10:24:23,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741926_1102 (size=12301) 2024-11-20T10:24:23,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T10:24:23,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:23,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:23,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:23,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:23,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T10:24:23,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098323257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:23,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098323266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098323266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:23,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098323266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:23,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T10:24:23,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:23,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:23,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:23,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:23,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:23,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/41ec57ad6d03442cbeeb873646485eaf 2024-11-20T10:24:23,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/678dfa75cc1647249430d445ff48a13b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/678dfa75cc1647249430d445ff48a13b 2024-11-20T10:24:23,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/678dfa75cc1647249430d445ff48a13b, entries=200, sequenceid=438, filesize=14.4 K 2024-11-20T10:24:23,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/f01b1c8a9c8b4583bc41c0bc0879c0a6 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/f01b1c8a9c8b4583bc41c0bc0879c0a6 2024-11-20T10:24:23,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/f01b1c8a9c8b4583bc41c0bc0879c0a6, entries=150, sequenceid=438, filesize=12.0 K 2024-11-20T10:24:23,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/41ec57ad6d03442cbeeb873646485eaf as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/41ec57ad6d03442cbeeb873646485eaf 2024-11-20T10:24:23,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/41ec57ad6d03442cbeeb873646485eaf, entries=150, sequenceid=438, filesize=12.0 K 2024-11-20T10:24:23,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 455386d186b9f64def53c239491339e4 in 893ms, sequenceid=438, compaction requested=true 2024-11-20T10:24:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:23,492 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:23,492 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:23,493 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:23,493 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:23,493 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/B is initiating minor compaction (all files) 2024-11-20T10:24:23,493 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/A is initiating minor compaction (all files) 2024-11-20T10:24:23,493 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/B in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:23,494 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/3879fd93cdae4b5cbdce541b529a5817, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/d4b77137d2974d068dceb53805d9472c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/f01b1c8a9c8b4583bc41c0bc0879c0a6] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=36.9 K 2024-11-20T10:24:23,494 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/A in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:23,494 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/0cecbfb8e710465c8eb0500245a5ea85, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/7eada38c94fc402487b634753c320858, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/678dfa75cc1647249430d445ff48a13b] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=39.3 K 2024-11-20T10:24:23,494 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3879fd93cdae4b5cbdce541b529a5817, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732098261144 2024-11-20T10:24:23,494 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cecbfb8e710465c8eb0500245a5ea85, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732098261144 2024-11-20T10:24:23,494 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d4b77137d2974d068dceb53805d9472c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732098261281 2024-11-20T10:24:23,494 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7eada38c94fc402487b634753c320858, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732098261281 2024-11-20T10:24:23,495 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f01b1c8a9c8b4583bc41c0bc0879c0a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732098261965 2024-11-20T10:24:23,495 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 678dfa75cc1647249430d445ff48a13b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732098261965 2024-11-20T10:24:23,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T10:24:23,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:23,497 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T10:24:23,503 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#B#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:23,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/6a6178d10d0a414d91592364f15e58e3 is 50, key is test_row_0/A:col10/1732098262626/Put/seqid=0 2024-11-20T10:24:23,504 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/80495bfd6c574e4099e3bea5381d15db is 50, key is test_row_0/B:col10/1732098261970/Put/seqid=0 2024-11-20T10:24:23,506 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#A#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:23,507 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/954359ae730843ff9924597a112ae2f5 is 50, key is test_row_0/A:col10/1732098261970/Put/seqid=0 2024-11-20T10:24:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T10:24:23,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741927_1103 (size=12301) 2024-11-20T10:24:23,534 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/6a6178d10d0a414d91592364f15e58e3 2024-11-20T10:24:23,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/86e5e8d88e1349a599739f327b40c83a is 50, key is test_row_0/B:col10/1732098262626/Put/seqid=0 2024-11-20T10:24:23,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741928_1104 (size=13323) 2024-11-20T10:24:23,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741929_1105 (size=13323) 2024-11-20T10:24:23,559 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/80495bfd6c574e4099e3bea5381d15db as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/80495bfd6c574e4099e3bea5381d15db 2024-11-20T10:24:23,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741930_1106 (size=12301) 2024-11-20T10:24:23,582 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/954359ae730843ff9924597a112ae2f5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/954359ae730843ff9924597a112ae2f5 2024-11-20T10:24:23,583 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/B of 455386d186b9f64def53c239491339e4 into 80495bfd6c574e4099e3bea5381d15db(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:23,583 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:23,583 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/B, priority=13, startTime=1732098263492; duration=0sec 2024-11-20T10:24:23,583 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:23,583 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:B 2024-11-20T10:24:23,584 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:23,586 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:23,586 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/C is initiating minor compaction (all files) 2024-11-20T10:24:23,587 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/C in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:23,587 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/8fcf61f174884c7a863590529bbd30d3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/931847685d114d438bd08635c8cbd234, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/41ec57ad6d03442cbeeb873646485eaf] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=36.9 K 2024-11-20T10:24:23,588 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fcf61f174884c7a863590529bbd30d3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732098261144 2024-11-20T10:24:23,588 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 931847685d114d438bd08635c8cbd234, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732098261281 2024-11-20T10:24:23,589 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 41ec57ad6d03442cbeeb873646485eaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732098261965 2024-11-20T10:24:23,593 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/A of 455386d186b9f64def53c239491339e4 into 954359ae730843ff9924597a112ae2f5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:23,593 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:23,593 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/A, priority=13, startTime=1732098263492; duration=0sec 2024-11-20T10:24:23,594 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:23,594 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:A 2024-11-20T10:24:23,600 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#C#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:23,600 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/2425d333ce214013a4c014f98c763ca9 is 50, key is test_row_0/C:col10/1732098261970/Put/seqid=0 2024-11-20T10:24:23,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741931_1107 (size=13323) 2024-11-20T10:24:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:23,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:23,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098323791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:23,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098323795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098323795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:23,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098323800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:23,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098323906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:23,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098323907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098323907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:23,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098323907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:23,974 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/86e5e8d88e1349a599739f327b40c83a 2024-11-20T10:24:23,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/80365bad2d2b4c3392ab472f23563762 is 50, key is test_row_0/C:col10/1732098262626/Put/seqid=0 2024-11-20T10:24:23,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741932_1108 (size=12301) 2024-11-20T10:24:24,015 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/2425d333ce214013a4c014f98c763ca9 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/2425d333ce214013a4c014f98c763ca9 2024-11-20T10:24:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T10:24:24,022 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/C of 455386d186b9f64def53c239491339e4 into 2425d333ce214013a4c014f98c763ca9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:24,022 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:24,022 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/C, priority=13, startTime=1732098263492; duration=0sec 2024-11-20T10:24:24,022 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:24,022 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:C 2024-11-20T10:24:24,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098324117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098324117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098324117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098324117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,390 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/80365bad2d2b4c3392ab472f23563762 2024-11-20T10:24:24,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/6a6178d10d0a414d91592364f15e58e3 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/6a6178d10d0a414d91592364f15e58e3 2024-11-20T10:24:24,400 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/6a6178d10d0a414d91592364f15e58e3, entries=150, sequenceid=455, filesize=12.0 K 2024-11-20T10:24:24,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/86e5e8d88e1349a599739f327b40c83a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/86e5e8d88e1349a599739f327b40c83a 2024-11-20T10:24:24,405 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/86e5e8d88e1349a599739f327b40c83a, entries=150, sequenceid=455, filesize=12.0 K 2024-11-20T10:24:24,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/80365bad2d2b4c3392ab472f23563762 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/80365bad2d2b4c3392ab472f23563762 2024-11-20T10:24:24,410 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/80365bad2d2b4c3392ab472f23563762, entries=150, sequenceid=455, filesize=12.0 K 2024-11-20T10:24:24,411 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 455386d186b9f64def53c239491339e4 in 914ms, sequenceid=455, compaction requested=false 2024-11-20T10:24:24,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:24,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:24,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-20T10:24:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-20T10:24:24,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-20T10:24:24,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5320 sec 2024-11-20T10:24:24,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.5370 sec 2024-11-20T10:24:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:24,429 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:24,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T10:24:24,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/f590bcfb284f415e81cff3c5efb6f3c4 is 50, key is test_row_0/A:col10/1732098264427/Put/seqid=0 2024-11-20T10:24:24,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741933_1109 (size=12301) 2024-11-20T10:24:24,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098324442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098324442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098324444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098324446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098324557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098324557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098324557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098324557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098324767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098324767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098324767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098324767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:24,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/f590bcfb284f415e81cff3c5efb6f3c4 2024-11-20T10:24:24,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/78e45b7bbd4c4cdf9d44fc007134b9bc is 50, key is test_row_0/B:col10/1732098264427/Put/seqid=0 2024-11-20T10:24:24,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741934_1110 (size=12301) 2024-11-20T10:24:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T10:24:25,036 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:25,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-20T10:24:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T10:24:25,038 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:25,039 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:25,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:25,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098325077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098325077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098325077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098325077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T10:24:25,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T10:24:25,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:25,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:25,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:25,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:25,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:25,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/78e45b7bbd4c4cdf9d44fc007134b9bc 2024-11-20T10:24:25,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/65470bee104e446b9c5393380417c3c6 is 50, key is test_row_0/C:col10/1732098264427/Put/seqid=0 2024-11-20T10:24:25,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741935_1111 (size=12301) 2024-11-20T10:24:25,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/65470bee104e446b9c5393380417c3c6 2024-11-20T10:24:25,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/f590bcfb284f415e81cff3c5efb6f3c4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/f590bcfb284f415e81cff3c5efb6f3c4 2024-11-20T10:24:25,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/f590bcfb284f415e81cff3c5efb6f3c4, entries=150, sequenceid=481, filesize=12.0 K 2024-11-20T10:24:25,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/78e45b7bbd4c4cdf9d44fc007134b9bc as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/78e45b7bbd4c4cdf9d44fc007134b9bc 2024-11-20T10:24:25,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/78e45b7bbd4c4cdf9d44fc007134b9bc, entries=150, sequenceid=481, filesize=12.0 K 2024-11-20T10:24:25,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/65470bee104e446b9c5393380417c3c6 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/65470bee104e446b9c5393380417c3c6 2024-11-20T10:24:25,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T10:24:25,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:25,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:25,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:25,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:25,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:25,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/65470bee104e446b9c5393380417c3c6, entries=150, sequenceid=481, filesize=12.0 K 2024-11-20T10:24:25,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 455386d186b9f64def53c239491339e4 in 921ms, sequenceid=481, compaction requested=true 2024-11-20T10:24:25,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:25,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:25,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:25,351 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:25,351 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:25,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:25,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:25,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:25,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:25,353 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:25,353 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/B is initiating minor compaction (all files) 2024-11-20T10:24:25,353 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/B in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:25,353 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/80495bfd6c574e4099e3bea5381d15db, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/86e5e8d88e1349a599739f327b40c83a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/78e45b7bbd4c4cdf9d44fc007134b9bc] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=37.0 K 2024-11-20T10:24:25,355 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:25,355 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 80495bfd6c574e4099e3bea5381d15db, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732098261965 2024-11-20T10:24:25,355 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/A is initiating minor compaction (all files) 2024-11-20T10:24:25,355 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/A in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:25,355 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/954359ae730843ff9924597a112ae2f5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/6a6178d10d0a414d91592364f15e58e3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/f590bcfb284f415e81cff3c5efb6f3c4] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=37.0 K 2024-11-20T10:24:25,356 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 86e5e8d88e1349a599739f327b40c83a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732098262624 2024-11-20T10:24:25,356 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 954359ae730843ff9924597a112ae2f5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732098261965 2024-11-20T10:24:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T10:24:25,357 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 78e45b7bbd4c4cdf9d44fc007134b9bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732098263788 2024-11-20T10:24:25,357 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a6178d10d0a414d91592364f15e58e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732098262624 2024-11-20T10:24:25,358 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f590bcfb284f415e81cff3c5efb6f3c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732098263788 2024-11-20T10:24:25,370 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#A#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:25,370 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/6eeb7f3bc95c4e9eaa20ef4fab3ac8b8 is 50, key is test_row_0/A:col10/1732098264427/Put/seqid=0 2024-11-20T10:24:25,372 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:25,373 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/1a357f6b790a4b22836d79cefe737fd7 is 50, key is test_row_0/B:col10/1732098264427/Put/seqid=0 2024-11-20T10:24:25,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741936_1112 (size=13425) 2024-11-20T10:24:25,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741937_1113 (size=13425) 2024-11-20T10:24:25,395 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/6eeb7f3bc95c4e9eaa20ef4fab3ac8b8 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/6eeb7f3bc95c4e9eaa20ef4fab3ac8b8 2024-11-20T10:24:25,400 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/1a357f6b790a4b22836d79cefe737fd7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1a357f6b790a4b22836d79cefe737fd7 2024-11-20T10:24:25,404 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/A of 455386d186b9f64def53c239491339e4 into 6eeb7f3bc95c4e9eaa20ef4fab3ac8b8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:25,404 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:25,404 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/A, priority=13, startTime=1732098265351; duration=0sec 2024-11-20T10:24:25,404 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:25,404 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:A 2024-11-20T10:24:25,404 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:25,406 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:25,407 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/C is initiating minor compaction (all files) 2024-11-20T10:24:25,407 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/C in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:25,407 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/2425d333ce214013a4c014f98c763ca9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/80365bad2d2b4c3392ab472f23563762, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/65470bee104e446b9c5393380417c3c6] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=37.0 K 2024-11-20T10:24:25,409 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/B of 455386d186b9f64def53c239491339e4 into 1a357f6b790a4b22836d79cefe737fd7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:25,409 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:25,409 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/B, priority=13, startTime=1732098265351; duration=0sec 2024-11-20T10:24:25,409 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:25,409 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2425d333ce214013a4c014f98c763ca9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732098261965 2024-11-20T10:24:25,409 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:B 2024-11-20T10:24:25,410 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80365bad2d2b4c3392ab472f23563762, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732098262624 2024-11-20T10:24:25,411 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65470bee104e446b9c5393380417c3c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732098263788 2024-11-20T10:24:25,424 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#C#compaction#101 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:25,425 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/d95751bbca834483959d35b9eed03110 is 50, key is test_row_0/C:col10/1732098264427/Put/seqid=0 2024-11-20T10:24:25,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741938_1114 (size=13425) 2024-11-20T10:24:25,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T10:24:25,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:25,502 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T10:24:25,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/c1eadb61df5246af882e8b784ae6606e is 50, key is test_row_0/A:col10/1732098264442/Put/seqid=0 2024-11-20T10:24:25,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741939_1115 (size=12301) 2024-11-20T10:24:25,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. as already flushing 2024-11-20T10:24:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:25,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098325606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098325610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098325611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098325612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T10:24:25,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098325716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098325717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098325717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098325727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,781 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-20T10:24:25,781 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:24:25,782 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-20T10:24:25,782 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:24:25,786 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:25,786 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:25,786 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:25,786 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:25,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:24:25,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:24:25,787 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-20T10:24:25,787 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:24:25,787 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:25,788 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:25,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:24:25,792 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-20T10:24:25,792 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:24:25,792 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:25,792 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:25,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:24:25,841 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/d95751bbca834483959d35b9eed03110 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/d95751bbca834483959d35b9eed03110 2024-11-20T10:24:25,848 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/C of 455386d186b9f64def53c239491339e4 into d95751bbca834483959d35b9eed03110(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:25,848 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:25,848 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/C, priority=13, startTime=1732098265351; duration=0sec 2024-11-20T10:24:25,848 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:25,848 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:C 2024-11-20T10:24:25,926 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/c1eadb61df5246af882e8b784ae6606e 2024-11-20T10:24:25,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098325927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098325927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098325927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:25,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098325937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:25,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/4755722f484e4ee1b44dc77f3800daca is 50, key is test_row_0/B:col10/1732098264442/Put/seqid=0 2024-11-20T10:24:25,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741940_1116 (size=12301) 2024-11-20T10:24:26,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1732098326097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:26,100 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_1' in 455386d186b9f64def53c239491339e4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:24:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T10:24:26,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:26,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098326238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:26,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098326239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098326239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:26,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:26,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:26,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:26,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098326246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:26,347 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/4755722f484e4ee1b44dc77f3800daca 2024-11-20T10:24:26,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/c74c6f643fe44df5adb9010c168f056b is 50, key is test_row_0/C:col10/1732098264442/Put/seqid=0 2024-11-20T10:24:26,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741941_1117 (size=12301) 2024-11-20T10:24:26,534 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T10:24:26,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:26,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33378 deadline: 1732098326747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33338 deadline: 1732098326747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:26,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33352 deadline: 1732098326747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:26,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1732098326756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:26,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:26,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:24:26,762 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/c74c6f643fe44df5adb9010c168f056b 2024-11-20T10:24:26,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/c1eadb61df5246af882e8b784ae6606e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/c1eadb61df5246af882e8b784ae6606e 2024-11-20T10:24:26,777 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/c1eadb61df5246af882e8b784ae6606e, entries=150, sequenceid=496, filesize=12.0 K 2024-11-20T10:24:26,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/4755722f484e4ee1b44dc77f3800daca as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/4755722f484e4ee1b44dc77f3800daca 2024-11-20T10:24:26,784 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/4755722f484e4ee1b44dc77f3800daca, entries=150, sequenceid=496, filesize=12.0 K 2024-11-20T10:24:26,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/c74c6f643fe44df5adb9010c168f056b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/c74c6f643fe44df5adb9010c168f056b 2024-11-20T10:24:26,789 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/c74c6f643fe44df5adb9010c168f056b, entries=150, sequenceid=496, filesize=12.0 K 2024-11-20T10:24:26,790 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 455386d186b9f64def53c239491339e4 in 1289ms, sequenceid=496, compaction requested=false 2024-11-20T10:24:26,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:26,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:26,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-20T10:24:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-20T10:24:26,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-20T10:24:26,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7520 sec 2024-11-20T10:24:26,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.7570 sec 2024-11-20T10:24:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T10:24:27,188 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 455386d186b9f64def53c239491339e4 2024-11-20T10:24:27,758 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:27,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T10:24:27,759 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-20T10:24:27,759 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:24:27,759 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:27,759 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:27,759 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:24:27,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/e486aa2ff30246a89c68351ee6bf6b5a is 50, key is test_row_0/A:col10/1732098267757/Put/seqid=0 2024-11-20T10:24:27,769 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-20T10:24:27,769 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-20T10:24:27,769 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:24:27,769 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:24:27,769 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:27,769 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:27,769 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:27,769 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:27,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:24:27,770 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-20T10:24:27,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:24:27,770 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:24:27,770 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:27,770 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:27,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:24:27,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741942_1118 (size=12301) 2024-11-20T10:24:28,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/e486aa2ff30246a89c68351ee6bf6b5a 2024-11-20T10:24:28,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/c897516fb3cd40468f87097b72b0a89f is 50, key is test_row_0/B:col10/1732098267757/Put/seqid=0 2024-11-20T10:24:28,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741943_1119 (size=12301) 2024-11-20T10:24:28,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/c897516fb3cd40468f87097b72b0a89f 2024-11-20T10:24:28,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/1857009e3d4c4f8da2f9c44ce1a63af6 is 50, key is test_row_0/C:col10/1732098267757/Put/seqid=0 2024-11-20T10:24:28,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741944_1120 (size=12301) 2024-11-20T10:24:29,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/1857009e3d4c4f8da2f9c44ce1a63af6 2024-11-20T10:24:29,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/e486aa2ff30246a89c68351ee6bf6b5a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/e486aa2ff30246a89c68351ee6bf6b5a 2024-11-20T10:24:29,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/e486aa2ff30246a89c68351ee6bf6b5a, entries=150, sequenceid=521, filesize=12.0 K 2024-11-20T10:24:29,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/c897516fb3cd40468f87097b72b0a89f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c897516fb3cd40468f87097b72b0a89f 2024-11-20T10:24:29,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c897516fb3cd40468f87097b72b0a89f, entries=150, sequenceid=521, filesize=12.0 K 2024-11-20T10:24:29,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/1857009e3d4c4f8da2f9c44ce1a63af6 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1857009e3d4c4f8da2f9c44ce1a63af6 2024-11-20T10:24:29,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1857009e3d4c4f8da2f9c44ce1a63af6, entries=150, sequenceid=521, filesize=12.0 K 2024-11-20T10:24:29,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=20.13 KB/20610 for 455386d186b9f64def53c239491339e4 in 1354ms, sequenceid=521, compaction requested=true 2024-11-20T10:24:29,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:29,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:29,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:29,112 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:29,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:29,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:29,112 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:29,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 455386d186b9f64def53c239491339e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:29,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:29,114 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:29,114 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:29,114 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/B is initiating minor compaction (all files) 2024-11-20T10:24:29,114 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/A is initiating minor compaction (all files) 2024-11-20T10:24:29,114 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/A in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:29,114 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/B in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:29,114 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1a357f6b790a4b22836d79cefe737fd7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/4755722f484e4ee1b44dc77f3800daca, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c897516fb3cd40468f87097b72b0a89f] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=37.1 K 2024-11-20T10:24:29,114 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/6eeb7f3bc95c4e9eaa20ef4fab3ac8b8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/c1eadb61df5246af882e8b784ae6606e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/e486aa2ff30246a89c68351ee6bf6b5a] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=37.1 K 2024-11-20T10:24:29,115 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a357f6b790a4b22836d79cefe737fd7, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732098263788 2024-11-20T10:24:29,115 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eeb7f3bc95c4e9eaa20ef4fab3ac8b8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732098263788 2024-11-20T10:24:29,115 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4755722f484e4ee1b44dc77f3800daca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1732098264435 2024-11-20T10:24:29,115 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c1eadb61df5246af882e8b784ae6606e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1732098264435 2024-11-20T10:24:29,115 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c897516fb3cd40468f87097b72b0a89f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732098265602 2024-11-20T10:24:29,115 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting e486aa2ff30246a89c68351ee6bf6b5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732098265602 2024-11-20T10:24:29,123 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#B#compaction#109 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:29,123 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#A#compaction#108 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:29,124 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/95ad7006654a42a5be7429a21eadd7a1 is 50, key is test_row_0/B:col10/1732098267757/Put/seqid=0 2024-11-20T10:24:29,124 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/80c8024aa9fc45fa99b4177948a6d7ff is 50, key is test_row_0/A:col10/1732098267757/Put/seqid=0 2024-11-20T10:24:29,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741946_1122 (size=13527) 2024-11-20T10:24:29,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741945_1121 (size=13527) 2024-11-20T10:24:29,535 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/80c8024aa9fc45fa99b4177948a6d7ff as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/80c8024aa9fc45fa99b4177948a6d7ff 2024-11-20T10:24:29,535 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/95ad7006654a42a5be7429a21eadd7a1 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/95ad7006654a42a5be7429a21eadd7a1 2024-11-20T10:24:29,543 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/B of 455386d186b9f64def53c239491339e4 into 95ad7006654a42a5be7429a21eadd7a1(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:29,543 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/A of 455386d186b9f64def53c239491339e4 into 80c8024aa9fc45fa99b4177948a6d7ff(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:29,543 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:29,543 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:29,543 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/B, priority=13, startTime=1732098269112; duration=0sec 2024-11-20T10:24:29,543 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/A, priority=13, startTime=1732098269112; duration=0sec 2024-11-20T10:24:29,543 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:29,543 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:B 2024-11-20T10:24:29,543 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:29,543 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:29,543 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:A 2024-11-20T10:24:29,544 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:29,545 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 455386d186b9f64def53c239491339e4/C is initiating minor compaction (all files) 2024-11-20T10:24:29,545 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 455386d186b9f64def53c239491339e4/C in TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:29,545 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/d95751bbca834483959d35b9eed03110, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/c74c6f643fe44df5adb9010c168f056b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1857009e3d4c4f8da2f9c44ce1a63af6] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp, totalSize=37.1 K 2024-11-20T10:24:29,545 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting d95751bbca834483959d35b9eed03110, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732098263788 2024-11-20T10:24:29,546 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c74c6f643fe44df5adb9010c168f056b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1732098264435 2024-11-20T10:24:29,546 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1857009e3d4c4f8da2f9c44ce1a63af6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732098265602 2024-11-20T10:24:29,558 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 455386d186b9f64def53c239491339e4#C#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:29,559 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/461a33a8579a42a48fcae53d72ae70f5 is 50, key is test_row_0/C:col10/1732098267757/Put/seqid=0 2024-11-20T10:24:29,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741947_1123 (size=13527) 2024-11-20T10:24:29,583 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/461a33a8579a42a48fcae53d72ae70f5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/461a33a8579a42a48fcae53d72ae70f5 2024-11-20T10:24:29,590 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 455386d186b9f64def53c239491339e4/C of 455386d186b9f64def53c239491339e4 into 461a33a8579a42a48fcae53d72ae70f5(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:29,591 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 455386d186b9f64def53c239491339e4: 2024-11-20T10:24:29,591 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., storeName=455386d186b9f64def53c239491339e4/C, priority=13, startTime=1732098269112; duration=0sec 2024-11-20T10:24:29,591 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:29,591 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 455386d186b9f64def53c239491339e4:C 2024-11-20T10:24:36,168 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-20T10:24:36,169 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:24:36,169 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:36,169 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:36,169 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:24:36,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T10:24:36,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-20T10:24:36,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-20T10:24:36,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-20T10:24:36,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-20T10:24:36,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-20T10:24:36,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T10:24:36,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3307 2024-11-20T10:24:36,170 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3252 2024-11-20T10:24:36,170 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T10:24:36,170 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1503 2024-11-20T10:24:36,170 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4506 rows 2024-11-20T10:24:36,170 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1493 2024-11-20T10:24:36,170 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4476 rows 2024-11-20T10:24:36,170 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T10:24:36,170 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T10:24:36,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:36,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:36,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:24:36,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T10:24:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T10:24:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T10:24:36,184 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098276183"}]},"ts":"1732098276183"} 2024-11-20T10:24:36,185 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T10:24:36,185 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T10:24:36,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T10:24:36,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=455386d186b9f64def53c239491339e4, UNASSIGN}] 2024-11-20T10:24:36,192 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=455386d186b9f64def53c239491339e4, UNASSIGN 2024-11-20T10:24:36,193 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=455386d186b9f64def53c239491339e4, regionState=CLOSING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:24:36,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=455386d186b9f64def53c239491339e4, UNASSIGN because future has completed 2024-11-20T10:24:36,196 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T10:24:36,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:24:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T10:24:36,357 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 455386d186b9f64def53c239491339e4 2024-11-20T10:24:36,358 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T10:24:36,360 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 455386d186b9f64def53c239491339e4, disabling compactions & flushes 2024-11-20T10:24:36,360 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:36,360 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:36,360 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. after waiting 0 ms 2024-11-20T10:24:36,360 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:36,361 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 455386d186b9f64def53c239491339e4 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-20T10:24:36,367 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/928e38fe74644106b342a04a563ba692 is 50, key is test_row_1/A:col10/1732098276167/Put/seqid=0 2024-11-20T10:24:36,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741948_1124 (size=9857) 2024-11-20T10:24:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T10:24:36,774 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/928e38fe74644106b342a04a563ba692 2024-11-20T10:24:36,792 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/8be29f4c68884e8e93061c94e3c5763d is 50, key is test_row_1/B:col10/1732098276167/Put/seqid=0 2024-11-20T10:24:36,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741949_1125 (size=9857) 2024-11-20T10:24:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T10:24:37,198 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/8be29f4c68884e8e93061c94e3c5763d 2024-11-20T10:24:37,212 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/9df17a1cb2b74d85a056edec1d9918e1 is 50, key is test_row_1/C:col10/1732098276167/Put/seqid=0 2024-11-20T10:24:37,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741950_1126 (size=9857) 2024-11-20T10:24:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T10:24:37,619 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/9df17a1cb2b74d85a056edec1d9918e1 2024-11-20T10:24:37,629 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/A/928e38fe74644106b342a04a563ba692 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/928e38fe74644106b342a04a563ba692 2024-11-20T10:24:37,634 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/928e38fe74644106b342a04a563ba692, entries=100, sequenceid=531, filesize=9.6 K 2024-11-20T10:24:37,635 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/B/8be29f4c68884e8e93061c94e3c5763d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/8be29f4c68884e8e93061c94e3c5763d 2024-11-20T10:24:37,641 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/8be29f4c68884e8e93061c94e3c5763d, entries=100, sequenceid=531, filesize=9.6 K 2024-11-20T10:24:37,642 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/.tmp/C/9df17a1cb2b74d85a056edec1d9918e1 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9df17a1cb2b74d85a056edec1d9918e1 2024-11-20T10:24:37,647 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9df17a1cb2b74d85a056edec1d9918e1, entries=100, sequenceid=531, filesize=9.6 K 2024-11-20T10:24:37,648 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 455386d186b9f64def53c239491339e4 in 1288ms, sequenceid=531, compaction requested=false 2024-11-20T10:24:37,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bf352c7c2c754fb693673e87b5e880f7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bd7ae4f3ac724c45897fb6d97e265b32, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/12743c97e39744f2a2e7e2500847ba5c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/1d6b77f46a6e473481a12d75f57bdd17, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bd40961b524b4b5c98d4d530a63c1b2c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/52a2983ea55d4ad0a4cbff64f72b3d6c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/9165702575de41ff8933c63c310a2d91, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/8a55b949c8674cc698ce7516449d2ab2, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2e5143a4f1b440b2bc70cd7525565435, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/ee495d7befe04de490cf7bf2575d08bd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/0b730db08cdb466989c53e76a2251b5f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/f19fa2b510034e249bd76705212e939c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/52295b9bdb0b4906800417b292bfabcc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/a940aa4b46a1408d9d374a0fd8ab9b6f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/1b6f24ef1d7a4af8bc766f7b3d020aa1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2f769256759d44cd9abaa19c394bf18e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/8ac88ef5107746a796abd3218fb04641, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/512e66c4af2e449a9775ee181f3d748b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/039e49b2da8748eba57b08e10f82d441, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bad639157b8f443eb4475bf2f6b2d036, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/78bf114bf0614253bae2f9940f8d7683, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/27cfc780c1d4410eb5bebf5baf2e3416, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/d3c549a0b7684172b91261bd8edc6ab5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2d43716fefb443e58cd108369600a76d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/77405886c8f74e5e9d78603f6559348d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/3cbfbfacdb4347ad838a6802ab4df067, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/0cecbfb8e710465c8eb0500245a5ea85, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/993dea8c9f364a75923af4974eae647a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/7eada38c94fc402487b634753c320858, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/678dfa75cc1647249430d445ff48a13b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/954359ae730843ff9924597a112ae2f5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/6a6178d10d0a414d91592364f15e58e3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/6eeb7f3bc95c4e9eaa20ef4fab3ac8b8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/f590bcfb284f415e81cff3c5efb6f3c4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/c1eadb61df5246af882e8b784ae6606e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/e486aa2ff30246a89c68351ee6bf6b5a] to archive 2024-11-20T10:24:37,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:24:37,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bf352c7c2c754fb693673e87b5e880f7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bf352c7c2c754fb693673e87b5e880f7 2024-11-20T10:24:37,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bd7ae4f3ac724c45897fb6d97e265b32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bd7ae4f3ac724c45897fb6d97e265b32 2024-11-20T10:24:37,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/12743c97e39744f2a2e7e2500847ba5c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/12743c97e39744f2a2e7e2500847ba5c 2024-11-20T10:24:37,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/1d6b77f46a6e473481a12d75f57bdd17 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/1d6b77f46a6e473481a12d75f57bdd17 2024-11-20T10:24:37,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bd40961b524b4b5c98d4d530a63c1b2c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bd40961b524b4b5c98d4d530a63c1b2c 2024-11-20T10:24:37,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/52a2983ea55d4ad0a4cbff64f72b3d6c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/52a2983ea55d4ad0a4cbff64f72b3d6c 2024-11-20T10:24:37,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/9165702575de41ff8933c63c310a2d91 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/9165702575de41ff8933c63c310a2d91 2024-11-20T10:24:37,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/8a55b949c8674cc698ce7516449d2ab2 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/8a55b949c8674cc698ce7516449d2ab2 2024-11-20T10:24:37,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2e5143a4f1b440b2bc70cd7525565435 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2e5143a4f1b440b2bc70cd7525565435 2024-11-20T10:24:37,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/ee495d7befe04de490cf7bf2575d08bd to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/ee495d7befe04de490cf7bf2575d08bd 2024-11-20T10:24:37,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/0b730db08cdb466989c53e76a2251b5f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/0b730db08cdb466989c53e76a2251b5f 2024-11-20T10:24:37,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/f19fa2b510034e249bd76705212e939c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/f19fa2b510034e249bd76705212e939c 2024-11-20T10:24:37,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/52295b9bdb0b4906800417b292bfabcc to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/52295b9bdb0b4906800417b292bfabcc 2024-11-20T10:24:37,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/a940aa4b46a1408d9d374a0fd8ab9b6f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/a940aa4b46a1408d9d374a0fd8ab9b6f 2024-11-20T10:24:37,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/1b6f24ef1d7a4af8bc766f7b3d020aa1 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/1b6f24ef1d7a4af8bc766f7b3d020aa1 2024-11-20T10:24:37,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2f769256759d44cd9abaa19c394bf18e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2f769256759d44cd9abaa19c394bf18e 2024-11-20T10:24:37,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/8ac88ef5107746a796abd3218fb04641 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/8ac88ef5107746a796abd3218fb04641 2024-11-20T10:24:37,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/512e66c4af2e449a9775ee181f3d748b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/512e66c4af2e449a9775ee181f3d748b 2024-11-20T10:24:37,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/039e49b2da8748eba57b08e10f82d441 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/039e49b2da8748eba57b08e10f82d441 2024-11-20T10:24:37,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bad639157b8f443eb4475bf2f6b2d036 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/bad639157b8f443eb4475bf2f6b2d036 2024-11-20T10:24:37,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/78bf114bf0614253bae2f9940f8d7683 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/78bf114bf0614253bae2f9940f8d7683 2024-11-20T10:24:37,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/27cfc780c1d4410eb5bebf5baf2e3416 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/27cfc780c1d4410eb5bebf5baf2e3416 2024-11-20T10:24:37,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/d3c549a0b7684172b91261bd8edc6ab5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/d3c549a0b7684172b91261bd8edc6ab5 2024-11-20T10:24:37,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2d43716fefb443e58cd108369600a76d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/2d43716fefb443e58cd108369600a76d 2024-11-20T10:24:37,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/77405886c8f74e5e9d78603f6559348d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/77405886c8f74e5e9d78603f6559348d 2024-11-20T10:24:37,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/3cbfbfacdb4347ad838a6802ab4df067 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/3cbfbfacdb4347ad838a6802ab4df067 2024-11-20T10:24:37,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/0cecbfb8e710465c8eb0500245a5ea85 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/0cecbfb8e710465c8eb0500245a5ea85 2024-11-20T10:24:37,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/993dea8c9f364a75923af4974eae647a to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/993dea8c9f364a75923af4974eae647a 2024-11-20T10:24:37,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/7eada38c94fc402487b634753c320858 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/7eada38c94fc402487b634753c320858 2024-11-20T10:24:37,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/678dfa75cc1647249430d445ff48a13b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/678dfa75cc1647249430d445ff48a13b 2024-11-20T10:24:37,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/954359ae730843ff9924597a112ae2f5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/954359ae730843ff9924597a112ae2f5 2024-11-20T10:24:37,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/6a6178d10d0a414d91592364f15e58e3 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/6a6178d10d0a414d91592364f15e58e3 2024-11-20T10:24:37,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/6eeb7f3bc95c4e9eaa20ef4fab3ac8b8 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/6eeb7f3bc95c4e9eaa20ef4fab3ac8b8 2024-11-20T10:24:37,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/f590bcfb284f415e81cff3c5efb6f3c4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/f590bcfb284f415e81cff3c5efb6f3c4 2024-11-20T10:24:37,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/c1eadb61df5246af882e8b784ae6606e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/c1eadb61df5246af882e8b784ae6606e 2024-11-20T10:24:37,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/e486aa2ff30246a89c68351ee6bf6b5a to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/e486aa2ff30246a89c68351ee6bf6b5a 2024-11-20T10:24:37,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1fc4a435afc1417493de4da4dd282e6f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1f57febc9b8646dc9a54dd13f3cf8638, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/eadaa23c704349cf95a92eabfd649dce, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/e8b19409ce6b4c7582cc25ff06d5d365, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/596c1778a141454da82eac3f8a806b1f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/9cb3ad7e5ba045a190804a580489ac44, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/cb83d9ee9cb24fbbae0366ba580a64ac, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/bf0edc5087584ad2a5f641a4c736f92d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/4559e6f10b51442c94324110bb36b3fb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c2b41641be0f423f90f6e37bf6a4b576, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/36d217b0bf57431e89618aaded207bfe, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/a6a3ba3c4ab84c2493438e1010d631cd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/b113bda782424af08077f261ce7ba480, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/5e1791da847b478799bb31d11c002ea0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/d58abea32932493b86feb6e3e4fd4e22, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/366c18f92b0e4aca823042c77335305e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/bfc03ebba6d94d158107a01e5043a32b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/0ca747363f654159ad4664275325bffb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/54d62cfdc629476c823abe710ad0b7fa, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/dc76c05f97ef4b4e9210f582e308217b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/360b57b089de4b538f9dbf77dce891b2, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/3919e4ac689046de9da21a9ae29f6506, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/8553ac7ae75640549f7123c1dd263043, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/420c921d21a14b20b20c64eab7ca6575, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c0d477de169f48ae8598623f1743b7f3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/a28a6ff5656e462795a619f1f53fffa4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/3879fd93cdae4b5cbdce541b529a5817, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/9417e20f543e4894930ab60c99cd481e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/d4b77137d2974d068dceb53805d9472c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/80495bfd6c574e4099e3bea5381d15db, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/f01b1c8a9c8b4583bc41c0bc0879c0a6, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/86e5e8d88e1349a599739f327b40c83a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1a357f6b790a4b22836d79cefe737fd7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/78e45b7bbd4c4cdf9d44fc007134b9bc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/4755722f484e4ee1b44dc77f3800daca, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c897516fb3cd40468f87097b72b0a89f] to archive 2024-11-20T10:24:37,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:24:37,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1fc4a435afc1417493de4da4dd282e6f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1fc4a435afc1417493de4da4dd282e6f 2024-11-20T10:24:37,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1f57febc9b8646dc9a54dd13f3cf8638 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1f57febc9b8646dc9a54dd13f3cf8638 2024-11-20T10:24:37,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/eadaa23c704349cf95a92eabfd649dce to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/eadaa23c704349cf95a92eabfd649dce 2024-11-20T10:24:37,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/e8b19409ce6b4c7582cc25ff06d5d365 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/e8b19409ce6b4c7582cc25ff06d5d365 2024-11-20T10:24:37,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/596c1778a141454da82eac3f8a806b1f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/596c1778a141454da82eac3f8a806b1f 2024-11-20T10:24:37,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/9cb3ad7e5ba045a190804a580489ac44 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/9cb3ad7e5ba045a190804a580489ac44 2024-11-20T10:24:37,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/cb83d9ee9cb24fbbae0366ba580a64ac to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/cb83d9ee9cb24fbbae0366ba580a64ac 2024-11-20T10:24:37,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/bf0edc5087584ad2a5f641a4c736f92d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/bf0edc5087584ad2a5f641a4c736f92d 2024-11-20T10:24:37,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/4559e6f10b51442c94324110bb36b3fb to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/4559e6f10b51442c94324110bb36b3fb 2024-11-20T10:24:37,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c2b41641be0f423f90f6e37bf6a4b576 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c2b41641be0f423f90f6e37bf6a4b576 2024-11-20T10:24:37,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/36d217b0bf57431e89618aaded207bfe to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/36d217b0bf57431e89618aaded207bfe 2024-11-20T10:24:37,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/a6a3ba3c4ab84c2493438e1010d631cd to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/a6a3ba3c4ab84c2493438e1010d631cd 2024-11-20T10:24:37,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/b113bda782424af08077f261ce7ba480 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/b113bda782424af08077f261ce7ba480 2024-11-20T10:24:37,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/5e1791da847b478799bb31d11c002ea0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/5e1791da847b478799bb31d11c002ea0 2024-11-20T10:24:37,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/d58abea32932493b86feb6e3e4fd4e22 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/d58abea32932493b86feb6e3e4fd4e22 2024-11-20T10:24:37,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/366c18f92b0e4aca823042c77335305e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/366c18f92b0e4aca823042c77335305e 2024-11-20T10:24:37,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/bfc03ebba6d94d158107a01e5043a32b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/bfc03ebba6d94d158107a01e5043a32b 2024-11-20T10:24:37,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/0ca747363f654159ad4664275325bffb to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/0ca747363f654159ad4664275325bffb 2024-11-20T10:24:37,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/54d62cfdc629476c823abe710ad0b7fa to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/54d62cfdc629476c823abe710ad0b7fa 2024-11-20T10:24:37,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/dc76c05f97ef4b4e9210f582e308217b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/dc76c05f97ef4b4e9210f582e308217b 2024-11-20T10:24:37,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/360b57b089de4b538f9dbf77dce891b2 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/360b57b089de4b538f9dbf77dce891b2 2024-11-20T10:24:37,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/3919e4ac689046de9da21a9ae29f6506 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/3919e4ac689046de9da21a9ae29f6506 2024-11-20T10:24:37,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/8553ac7ae75640549f7123c1dd263043 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/8553ac7ae75640549f7123c1dd263043 2024-11-20T10:24:37,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/420c921d21a14b20b20c64eab7ca6575 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/420c921d21a14b20b20c64eab7ca6575 2024-11-20T10:24:37,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c0d477de169f48ae8598623f1743b7f3 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c0d477de169f48ae8598623f1743b7f3 2024-11-20T10:24:37,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/a28a6ff5656e462795a619f1f53fffa4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/a28a6ff5656e462795a619f1f53fffa4 2024-11-20T10:24:37,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/3879fd93cdae4b5cbdce541b529a5817 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/3879fd93cdae4b5cbdce541b529a5817 2024-11-20T10:24:37,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/9417e20f543e4894930ab60c99cd481e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/9417e20f543e4894930ab60c99cd481e 2024-11-20T10:24:37,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/d4b77137d2974d068dceb53805d9472c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/d4b77137d2974d068dceb53805d9472c 2024-11-20T10:24:37,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/80495bfd6c574e4099e3bea5381d15db to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/80495bfd6c574e4099e3bea5381d15db 2024-11-20T10:24:37,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/f01b1c8a9c8b4583bc41c0bc0879c0a6 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/f01b1c8a9c8b4583bc41c0bc0879c0a6 2024-11-20T10:24:37,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/86e5e8d88e1349a599739f327b40c83a to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/86e5e8d88e1349a599739f327b40c83a 2024-11-20T10:24:37,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1a357f6b790a4b22836d79cefe737fd7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/1a357f6b790a4b22836d79cefe737fd7 2024-11-20T10:24:37,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/78e45b7bbd4c4cdf9d44fc007134b9bc to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/78e45b7bbd4c4cdf9d44fc007134b9bc 2024-11-20T10:24:37,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/4755722f484e4ee1b44dc77f3800daca to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/4755722f484e4ee1b44dc77f3800daca 2024-11-20T10:24:37,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c897516fb3cd40468f87097b72b0a89f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/c897516fb3cd40468f87097b72b0a89f 2024-11-20T10:24:37,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/95167c307c1b4509bf4ad79896653625, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9efe30daff6145768a54524b49b207b5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9849114ada404d7b82551d825f09d772, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/318508ad74f04099824e93fe3fd61e79, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1ee1378de1304428a3c32f1c35d642ef, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/09c9d5458f2f44e0b2eddef0824fb9ed, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/70d18833a5f142749e6cd3f09fbc571b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1b636f12a83a425b9eb4a729d24033f2, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/fd976d3d3fc14906af51d145886a6106, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/babc86afd0eb4adfb8b48bad5d19ba1d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/4600620715e249d58170d1100fd2bb24, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/179db8814c2e422095defb10326c372e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/8a66abdca7434301b86e42ffc0796855, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/f04794e800e74367b9b14b72f06e08cf, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9158d0592a0143d7960fd174c3db73ba, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/72bec96218494f738baeea6de1226854, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1b7edb8b85274d3aa930cd457a038b84, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9b5b13a52f264076a3e33eb2d869b370, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/cc689d92e95f438f8c94b7cae42d30e1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/4d7cd14052b2455d9ffb035cc5fd1626, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/7ac0785b771947048afdfa6617a307a8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/6ac41fa359714d769d8733639c754510, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1dc1a78941a449acb245d23444d3500b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/c928fea9922b428eac1fd5b8a7afe1d0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/3de6b1c0b7b04589a255dab778d8b54a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/cc7d092ed1254676b404e203ddfedf07, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/8fcf61f174884c7a863590529bbd30d3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/a939cae2cddd4e78b00f7409321663ce, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/931847685d114d438bd08635c8cbd234, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/2425d333ce214013a4c014f98c763ca9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/41ec57ad6d03442cbeeb873646485eaf, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/80365bad2d2b4c3392ab472f23563762, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/d95751bbca834483959d35b9eed03110, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/65470bee104e446b9c5393380417c3c6, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/c74c6f643fe44df5adb9010c168f056b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1857009e3d4c4f8da2f9c44ce1a63af6] to archive 2024-11-20T10:24:37,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:24:37,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/95167c307c1b4509bf4ad79896653625 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/95167c307c1b4509bf4ad79896653625 2024-11-20T10:24:37,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9efe30daff6145768a54524b49b207b5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9efe30daff6145768a54524b49b207b5 2024-11-20T10:24:37,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9849114ada404d7b82551d825f09d772 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9849114ada404d7b82551d825f09d772 2024-11-20T10:24:37,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/318508ad74f04099824e93fe3fd61e79 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/318508ad74f04099824e93fe3fd61e79 2024-11-20T10:24:37,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1ee1378de1304428a3c32f1c35d642ef to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1ee1378de1304428a3c32f1c35d642ef 2024-11-20T10:24:37,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/09c9d5458f2f44e0b2eddef0824fb9ed to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/09c9d5458f2f44e0b2eddef0824fb9ed 2024-11-20T10:24:37,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/70d18833a5f142749e6cd3f09fbc571b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/70d18833a5f142749e6cd3f09fbc571b 2024-11-20T10:24:37,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1b636f12a83a425b9eb4a729d24033f2 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1b636f12a83a425b9eb4a729d24033f2 2024-11-20T10:24:37,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/fd976d3d3fc14906af51d145886a6106 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/fd976d3d3fc14906af51d145886a6106 2024-11-20T10:24:37,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/babc86afd0eb4adfb8b48bad5d19ba1d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/babc86afd0eb4adfb8b48bad5d19ba1d 2024-11-20T10:24:37,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/4600620715e249d58170d1100fd2bb24 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/4600620715e249d58170d1100fd2bb24 2024-11-20T10:24:37,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/179db8814c2e422095defb10326c372e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/179db8814c2e422095defb10326c372e 2024-11-20T10:24:37,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/8a66abdca7434301b86e42ffc0796855 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/8a66abdca7434301b86e42ffc0796855 2024-11-20T10:24:37,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/f04794e800e74367b9b14b72f06e08cf to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/f04794e800e74367b9b14b72f06e08cf 2024-11-20T10:24:37,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9158d0592a0143d7960fd174c3db73ba to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9158d0592a0143d7960fd174c3db73ba 2024-11-20T10:24:37,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/72bec96218494f738baeea6de1226854 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/72bec96218494f738baeea6de1226854 2024-11-20T10:24:37,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1b7edb8b85274d3aa930cd457a038b84 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1b7edb8b85274d3aa930cd457a038b84 2024-11-20T10:24:37,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9b5b13a52f264076a3e33eb2d869b370 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9b5b13a52f264076a3e33eb2d869b370 2024-11-20T10:24:37,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/cc689d92e95f438f8c94b7cae42d30e1 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/cc689d92e95f438f8c94b7cae42d30e1 2024-11-20T10:24:37,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/4d7cd14052b2455d9ffb035cc5fd1626 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/4d7cd14052b2455d9ffb035cc5fd1626 2024-11-20T10:24:37,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/7ac0785b771947048afdfa6617a307a8 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/7ac0785b771947048afdfa6617a307a8 2024-11-20T10:24:37,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/6ac41fa359714d769d8733639c754510 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/6ac41fa359714d769d8733639c754510 2024-11-20T10:24:37,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1dc1a78941a449acb245d23444d3500b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1dc1a78941a449acb245d23444d3500b 2024-11-20T10:24:37,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/c928fea9922b428eac1fd5b8a7afe1d0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/c928fea9922b428eac1fd5b8a7afe1d0 2024-11-20T10:24:37,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/3de6b1c0b7b04589a255dab778d8b54a to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/3de6b1c0b7b04589a255dab778d8b54a 2024-11-20T10:24:37,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/cc7d092ed1254676b404e203ddfedf07 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/cc7d092ed1254676b404e203ddfedf07 2024-11-20T10:24:37,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/8fcf61f174884c7a863590529bbd30d3 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/8fcf61f174884c7a863590529bbd30d3 2024-11-20T10:24:37,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/a939cae2cddd4e78b00f7409321663ce to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/a939cae2cddd4e78b00f7409321663ce 2024-11-20T10:24:37,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/931847685d114d438bd08635c8cbd234 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/931847685d114d438bd08635c8cbd234 2024-11-20T10:24:37,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/2425d333ce214013a4c014f98c763ca9 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/2425d333ce214013a4c014f98c763ca9 2024-11-20T10:24:37,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/41ec57ad6d03442cbeeb873646485eaf to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/41ec57ad6d03442cbeeb873646485eaf 2024-11-20T10:24:37,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/80365bad2d2b4c3392ab472f23563762 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/80365bad2d2b4c3392ab472f23563762 2024-11-20T10:24:37,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/d95751bbca834483959d35b9eed03110 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/d95751bbca834483959d35b9eed03110 2024-11-20T10:24:37,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/65470bee104e446b9c5393380417c3c6 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/65470bee104e446b9c5393380417c3c6 2024-11-20T10:24:37,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/c74c6f643fe44df5adb9010c168f056b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/c74c6f643fe44df5adb9010c168f056b 2024-11-20T10:24:37,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1857009e3d4c4f8da2f9c44ce1a63af6 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/1857009e3d4c4f8da2f9c44ce1a63af6 2024-11-20T10:24:37,802 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/recovered.edits/534.seqid, newMaxSeqId=534, maxSeqId=1 2024-11-20T10:24:37,803 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. 2024-11-20T10:24:37,803 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 455386d186b9f64def53c239491339e4: Waiting for close lock at 1732098276359Running coprocessor pre-close hooks at 1732098276360 (+1 ms)Disabling compacts and flushes for region at 1732098276360Disabling writes for close at 1732098276360Obtaining lock to block concurrent updates at 1732098276361 (+1 ms)Preparing flush snapshotting stores in 455386d186b9f64def53c239491339e4 at 1732098276361Finished memstore snapshotting TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732098276361Flushing stores of TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4. at 1732098276362 (+1 ms)Flushing 455386d186b9f64def53c239491339e4/A: creating writer at 1732098276362Flushing 455386d186b9f64def53c239491339e4/A: appending metadata at 1732098276366 (+4 ms)Flushing 455386d186b9f64def53c239491339e4/A: closing flushed file at 1732098276366Flushing 455386d186b9f64def53c239491339e4/B: creating writer at 1732098276785 (+419 ms)Flushing 455386d186b9f64def53c239491339e4/B: appending metadata at 1732098276791 (+6 ms)Flushing 455386d186b9f64def53c239491339e4/B: closing flushed file at 1732098276791Flushing 455386d186b9f64def53c239491339e4/C: creating writer at 1732098277208 (+417 ms)Flushing 455386d186b9f64def53c239491339e4/C: appending metadata at 1732098277212 (+4 ms)Flushing 455386d186b9f64def53c239491339e4/C: closing flushed file at 1732098277212Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3249a5bb: reopening flushed file at 1732098277628 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e22a5c1: reopening flushed file at 1732098277634 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@547e6a52: reopening flushed file at 1732098277641 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 455386d186b9f64def53c239491339e4 in 1288ms, sequenceid=531, compaction requested=false at 1732098277648 (+7 ms)Writing region close event to WAL at 1732098277798 (+150 ms)Running coprocessor post-close hooks at 1732098277803 (+5 ms)Closed at 1732098277803 2024-11-20T10:24:37,806 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 455386d186b9f64def53c239491339e4 2024-11-20T10:24:37,806 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=455386d186b9f64def53c239491339e4, regionState=CLOSED 2024-11-20T10:24:37,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:24:37,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-20T10:24:37,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 455386d186b9f64def53c239491339e4, server=8c533afd331c,35833,1732098238678 in 1.6130 sec 2024-11-20T10:24:37,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-20T10:24:37,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=455386d186b9f64def53c239491339e4, UNASSIGN in 1.6200 sec 2024-11-20T10:24:37,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-20T10:24:37,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6270 sec 2024-11-20T10:24:37,818 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098277817"}]},"ts":"1732098277817"} 2024-11-20T10:24:37,820 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T10:24:37,820 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T10:24:37,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6440 sec 2024-11-20T10:24:37,887 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T10:24:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T10:24:38,328 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:38,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T10:24:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:24:38,342 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:24:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T10:24:38,344 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:24:38,352 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4 2024-11-20T10:24:38,356 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/recovered.edits] 2024-11-20T10:24:38,359 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/80c8024aa9fc45fa99b4177948a6d7ff to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/80c8024aa9fc45fa99b4177948a6d7ff 2024-11-20T10:24:38,360 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/928e38fe74644106b342a04a563ba692 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/A/928e38fe74644106b342a04a563ba692 2024-11-20T10:24:38,364 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/8be29f4c68884e8e93061c94e3c5763d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/8be29f4c68884e8e93061c94e3c5763d 2024-11-20T10:24:38,365 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/95ad7006654a42a5be7429a21eadd7a1 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/B/95ad7006654a42a5be7429a21eadd7a1 2024-11-20T10:24:38,368 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/461a33a8579a42a48fcae53d72ae70f5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/461a33a8579a42a48fcae53d72ae70f5 2024-11-20T10:24:38,370 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9df17a1cb2b74d85a056edec1d9918e1 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/C/9df17a1cb2b74d85a056edec1d9918e1 2024-11-20T10:24:38,374 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/recovered.edits/534.seqid to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4/recovered.edits/534.seqid 2024-11-20T10:24:38,375 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/455386d186b9f64def53c239491339e4 2024-11-20T10:24:38,375 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T10:24:38,382 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:24:38,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-20T10:24:38,392 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T10:24:38,427 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T10:24:38,428 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:24:38,428 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T10:24:38,429 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732098278428"}]},"ts":"9223372036854775807"} 2024-11-20T10:24:38,433 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T10:24:38,433 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 455386d186b9f64def53c239491339e4, NAME => 'TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T10:24:38,434 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T10:24:38,434 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732098278434"}]},"ts":"9223372036854775807"} 2024-11-20T10:24:38,438 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T10:24:38,439 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:24:38,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 104 msec 2024-11-20T10:24:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T10:24:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T10:24:38,447 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:38,457 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: RS:0;8c533afd331c:35833-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/8c533afd331c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-611424962_22 at /127.0.0.1:46182 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=474 (was 403) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8436 (was 9450) 2024-11-20T10:24:38,465 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=474, ProcessCount=11, AvailableMemoryMB=8435 2024-11-20T10:24:38,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T10:24:38,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T10:24:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T10:24:38,469 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T10:24:38,469 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:38,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-20T10:24:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T10:24:38,470 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T10:24:38,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741951_1127 (size=959) 2024-11-20T10:24:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T10:24:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T10:24:38,885 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31 2024-11-20T10:24:38,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741952_1128 (size=53) 2024-11-20T10:24:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T10:24:39,294 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:24:39,294 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a4b00f80e7a706f41be54575cc7fbb32, disabling compactions & flushes 2024-11-20T10:24:39,294 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:39,294 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:39,294 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. after waiting 0 ms 2024-11-20T10:24:39,294 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:39,294 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:39,294 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a4b00f80e7a706f41be54575cc7fbb32: Waiting for close lock at 1732098279294Disabling compacts and flushes for region at 1732098279294Disabling writes for close at 1732098279294Writing region close event to WAL at 1732098279294Closed at 1732098279294 2024-11-20T10:24:39,296 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T10:24:39,296 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732098279296"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732098279296"}]},"ts":"1732098279296"} 2024-11-20T10:24:39,298 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T10:24:39,299 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T10:24:39,299 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098279299"}]},"ts":"1732098279299"} 2024-11-20T10:24:39,301 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T10:24:39,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4b00f80e7a706f41be54575cc7fbb32, ASSIGN}] 2024-11-20T10:24:39,302 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4b00f80e7a706f41be54575cc7fbb32, ASSIGN 2024-11-20T10:24:39,303 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4b00f80e7a706f41be54575cc7fbb32, ASSIGN; state=OFFLINE, location=8c533afd331c,35833,1732098238678; forceNewPlan=false, retain=false 2024-11-20T10:24:39,454 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=a4b00f80e7a706f41be54575cc7fbb32, regionState=OPENING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:24:39,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4b00f80e7a706f41be54575cc7fbb32, ASSIGN because future has completed 2024-11-20T10:24:39,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:24:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T10:24:39,619 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:39,619 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:24:39,620 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:39,620 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:24:39,620 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:39,620 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:39,622 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:39,625 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4b00f80e7a706f41be54575cc7fbb32 columnFamilyName A 2024-11-20T10:24:39,626 DEBUG [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:39,626 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] regionserver.HStore(327): Store=a4b00f80e7a706f41be54575cc7fbb32/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:24:39,627 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:39,628 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4b00f80e7a706f41be54575cc7fbb32 columnFamilyName B 2024-11-20T10:24:39,629 DEBUG [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:39,629 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] regionserver.HStore(327): Store=a4b00f80e7a706f41be54575cc7fbb32/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:24:39,629 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:39,631 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4b00f80e7a706f41be54575cc7fbb32 columnFamilyName C 2024-11-20T10:24:39,631 DEBUG [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:39,632 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] regionserver.HStore(327): Store=a4b00f80e7a706f41be54575cc7fbb32/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:24:39,632 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:39,633 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:39,634 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:39,635 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:39,635 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:39,636 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T10:24:39,638 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:39,642 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:24:39,643 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened a4b00f80e7a706f41be54575cc7fbb32; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58991020, jitterRate=-0.12096530199050903}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T10:24:39,643 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:39,644 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for a4b00f80e7a706f41be54575cc7fbb32: Running coprocessor pre-open hook at 1732098279620Writing region info on filesystem at 1732098279620Initializing all the Stores at 1732098279622 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098279622Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098279622Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098279622Cleaning up temporary data from old regions at 1732098279635 (+13 ms)Running coprocessor post-open hooks at 1732098279643 (+8 ms)Region opened successfully at 1732098279644 (+1 ms) 2024-11-20T10:24:39,645 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., pid=32, masterSystemTime=1732098279611 2024-11-20T10:24:39,647 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:39,648 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:39,648 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=a4b00f80e7a706f41be54575cc7fbb32, regionState=OPEN, openSeqNum=2, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:24:39,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:24:39,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-20T10:24:39,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 in 194 msec 2024-11-20T10:24:39,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-20T10:24:39,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4b00f80e7a706f41be54575cc7fbb32, ASSIGN in 352 msec 2024-11-20T10:24:39,655 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T10:24:39,656 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098279655"}]},"ts":"1732098279655"} 2024-11-20T10:24:39,657 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T10:24:39,658 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T10:24:39,659 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1910 sec 2024-11-20T10:24:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T10:24:40,618 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:40,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168ecd40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:40,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:40,622 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:40,625 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:40,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:40,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:40,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@354a8bc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:40,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:40,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:40,630 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:40,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204753a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:40,632 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:40,636 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:40,638 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T10:24:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 8c533afd331c,33867,1732098237476 2024-11-20T10:24:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@67dee1fe 2024-11-20T10:24:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T10:24:40,640 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T10:24:40,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T10:24:40,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T10:24:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T10:24:40,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741953_1129 (size=995) 2024-11-20T10:24:41,064 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-20T10:24:41,064 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-20T10:24:41,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T10:24:41,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4b00f80e7a706f41be54575cc7fbb32, REOPEN/MOVE}] 2024-11-20T10:24:41,080 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4b00f80e7a706f41be54575cc7fbb32, REOPEN/MOVE 2024-11-20T10:24:41,081 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a4b00f80e7a706f41be54575cc7fbb32, regionState=CLOSING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:24:41,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4b00f80e7a706f41be54575cc7fbb32, REOPEN/MOVE because future has completed 2024-11-20T10:24:41,083 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T10:24:41,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:24:41,238 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,238 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T10:24:41,238 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing a4b00f80e7a706f41be54575cc7fbb32, disabling compactions & flushes 2024-11-20T10:24:41,239 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:41,239 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:41,239 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. after waiting 0 ms 2024-11-20T10:24:41,239 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:41,250 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T10:24:41,251 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:41,251 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for a4b00f80e7a706f41be54575cc7fbb32: Waiting for close lock at 1732098281238Running coprocessor pre-close hooks at 1732098281238Disabling compacts and flushes for region at 1732098281238Disabling writes for close at 1732098281239 (+1 ms)Writing region close event to WAL at 1732098281240 (+1 ms)Running coprocessor post-close hooks at 1732098281251 (+11 ms)Closed at 1732098281251 2024-11-20T10:24:41,251 WARN [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: a4b00f80e7a706f41be54575cc7fbb32 to self. 2024-11-20T10:24:41,253 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,254 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a4b00f80e7a706f41be54575cc7fbb32, regionState=CLOSED 2024-11-20T10:24:41,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:24:41,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-20T10:24:41,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 in 172 msec 2024-11-20T10:24:41,258 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4b00f80e7a706f41be54575cc7fbb32, REOPEN/MOVE; state=CLOSED, location=8c533afd331c,35833,1732098238678; forceNewPlan=false, retain=true 2024-11-20T10:24:41,409 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a4b00f80e7a706f41be54575cc7fbb32, regionState=OPENING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:24:41,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4b00f80e7a706f41be54575cc7fbb32, REOPEN/MOVE because future has completed 2024-11-20T10:24:41,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:24:41,572 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:41,573 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:24:41,573 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,574 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:24:41,574 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,574 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,579 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,585 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4b00f80e7a706f41be54575cc7fbb32 columnFamilyName A 2024-11-20T10:24:41,587 DEBUG [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:41,587 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] regionserver.HStore(327): Store=a4b00f80e7a706f41be54575cc7fbb32/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:24:41,588 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,588 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4b00f80e7a706f41be54575cc7fbb32 columnFamilyName B 2024-11-20T10:24:41,588 DEBUG [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:41,589 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] regionserver.HStore(327): Store=a4b00f80e7a706f41be54575cc7fbb32/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:24:41,589 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,590 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4b00f80e7a706f41be54575cc7fbb32 columnFamilyName C 2024-11-20T10:24:41,590 DEBUG [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:41,590 INFO [StoreOpener-a4b00f80e7a706f41be54575cc7fbb32-1 {}] regionserver.HStore(327): Store=a4b00f80e7a706f41be54575cc7fbb32/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:24:41,590 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,591 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,592 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,593 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,593 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,594 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T10:24:41,595 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,596 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened a4b00f80e7a706f41be54575cc7fbb32; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59309327, jitterRate=-0.11622215807437897}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T10:24:41,596 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,597 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for a4b00f80e7a706f41be54575cc7fbb32: Running coprocessor pre-open hook at 1732098281574Writing region info on filesystem at 1732098281574Initializing all the Stores at 1732098281576 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098281576Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098281576Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098281576Cleaning up temporary data from old regions at 1732098281593 (+17 ms)Running coprocessor post-open hooks at 1732098281596 (+3 ms)Region opened successfully at 1732098281597 (+1 ms) 2024-11-20T10:24:41,598 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., pid=37, masterSystemTime=1732098281568 2024-11-20T10:24:41,600 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:41,600 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:41,601 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a4b00f80e7a706f41be54575cc7fbb32, regionState=OPEN, openSeqNum=5, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:24:41,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:24:41,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-20T10:24:41,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 in 190 msec 2024-11-20T10:24:41,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-20T10:24:41,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4b00f80e7a706f41be54575cc7fbb32, REOPEN/MOVE in 526 msec 2024-11-20T10:24:41,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-20T10:24:41,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 537 msec 2024-11-20T10:24:41,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 964 msec 2024-11-20T10:24:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-20T10:24:41,615 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:41,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@552fb1a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,616 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:41,616 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:41,617 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ed6f17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:41,618 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b87891, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18f01ca4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:41,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:41,620 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2149b6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:41,622 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:41,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5965295d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a283555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,623 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:41,623 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:41,624 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:41,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:41,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:41,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145bd131, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:41,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:41,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:41,625 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:41,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e9117a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ef6e97c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:41,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:41,627 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1611af17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:41,629 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:41,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5853fc53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14984fac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,630 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:41,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:41,631 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26e48fb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:41,632 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:41,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce05b6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a21ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:41,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:41,635 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dafd40a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:41,636 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a64d063, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f62ef44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:41,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:41,638 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@990b0ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:41,639 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46c9390b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f60d5da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:41,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:41,641 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:41,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:41,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13021d37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:41,642 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33122ebe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139f617f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:24:41,644 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:24:41,644 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:24:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:24:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:24:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43f539cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:24:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:24:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:24:41,646 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:24:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555e7ba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:24:41,647 DEBUG [Thread-661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:41,647 DEBUG [Thread-667 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:41,647 DEBUG [Thread-656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:41,647 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:41,648 DEBUG [Thread-654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:41,648 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:41,648 DEBUG [Thread-665 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:41,649 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:41,649 DEBUG [Thread-663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:24:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:41,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:41,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:41,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:41,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:41,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:41,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:24:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:24:41,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:41,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:41,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:41,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:24:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-20T10:24:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T10:24:41,654 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:41,655 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:41,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:24:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:24:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:24:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:24:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:24:41,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:24:41,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:24:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:24:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:24:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,676 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:41,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:24:41,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c58ae73122284a1aa9d0e1c8392401a0_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098281676/Put/seqid=0 2024-11-20T10:24:41,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098341724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098341726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:41,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741954_1130 (size=12154) 2024-11-20T10:24:41,742 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:41,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098341731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:41,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:41,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098341733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:41,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:41,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098341735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:41,751 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c58ae73122284a1aa9d0e1c8392401a0_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c58ae73122284a1aa9d0e1c8392401a0_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:41,752 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/aa80b20109c744609b84e7eb6a49ca4f, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T10:24:41,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/aa80b20109c744609b84e7eb6a49ca4f is 175, key is test_row_0/A:col10/1732098281676/Put/seqid=0 2024-11-20T10:24:41,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741955_1131 (size=30955) 2024-11-20T10:24:41,777 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/aa80b20109c744609b84e7eb6a49ca4f 2024-11-20T10:24:41,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/f1e0e20571cb4b6f86c3223a23b3cfb4 is 50, key is test_row_0/B:col10/1732098281676/Put/seqid=0 2024-11-20T10:24:41,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T10:24:41,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:41,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:41,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:41,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:41,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:41,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741956_1132 (size=12001) 2024-11-20T10:24:41,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/f1e0e20571cb4b6f86c3223a23b3cfb4 2024-11-20T10:24:41,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098341847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:41,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098341847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:41,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:41,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098341847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:41,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:41,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/350c4f9f759d49ff880320fe83915e16 is 50, key is test_row_0/C:col10/1732098281676/Put/seqid=0 2024-11-20T10:24:41,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098341857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:41,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:41,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098341857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:41,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741957_1133 (size=12001) 2024-11-20T10:24:41,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/350c4f9f759d49ff880320fe83915e16 2024-11-20T10:24:41,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/aa80b20109c744609b84e7eb6a49ca4f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/aa80b20109c744609b84e7eb6a49ca4f 2024-11-20T10:24:41,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/aa80b20109c744609b84e7eb6a49ca4f, entries=150, sequenceid=17, filesize=30.2 K 2024-11-20T10:24:41,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/f1e0e20571cb4b6f86c3223a23b3cfb4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f1e0e20571cb4b6f86c3223a23b3cfb4 2024-11-20T10:24:41,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f1e0e20571cb4b6f86c3223a23b3cfb4, entries=150, sequenceid=17, filesize=11.7 K 2024-11-20T10:24:41,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/350c4f9f759d49ff880320fe83915e16 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/350c4f9f759d49ff880320fe83915e16 2024-11-20T10:24:41,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/350c4f9f759d49ff880320fe83915e16, entries=150, sequenceid=17, filesize=11.7 K 2024-11-20T10:24:41,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for a4b00f80e7a706f41be54575cc7fbb32 in 219ms, sequenceid=17, compaction requested=false 2024-11-20T10:24:41,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T10:24:41,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T10:24:41,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:41,970 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T10:24:41,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208acf8086ba7f4a79a94d838b9427bedc_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098281731/Put/seqid=0 2024-11-20T10:24:41,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741958_1134 (size=12154) 2024-11-20T10:24:41,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:42,005 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208acf8086ba7f4a79a94d838b9427bedc_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208acf8086ba7f4a79a94d838b9427bedc_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/1321e7484d9e4f8caadb491ead79c432, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:42,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/1321e7484d9e4f8caadb491ead79c432 is 175, key is test_row_0/A:col10/1732098281731/Put/seqid=0 2024-11-20T10:24:42,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741959_1135 (size=30955) 2024-11-20T10:24:42,021 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/1321e7484d9e4f8caadb491ead79c432 2024-11-20T10:24:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/bd98e428b9374a3dbc7610e7ecf45741 is 50, key is test_row_0/B:col10/1732098281731/Put/seqid=0 2024-11-20T10:24:42,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741960_1136 (size=12001) 2024-11-20T10:24:42,049 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/bd98e428b9374a3dbc7610e7ecf45741 2024-11-20T10:24:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:42,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:42,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/9d9fedfda1164f8781070cca9eb8eca5 is 50, key is test_row_0/C:col10/1732098281731/Put/seqid=0 2024-11-20T10:24:42,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741961_1137 (size=12001) 2024-11-20T10:24:42,082 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/9d9fedfda1164f8781070cca9eb8eca5 2024-11-20T10:24:42,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/1321e7484d9e4f8caadb491ead79c432 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/1321e7484d9e4f8caadb491ead79c432 2024-11-20T10:24:42,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098342077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098342078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098342079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,098 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/1321e7484d9e4f8caadb491ead79c432, entries=150, sequenceid=41, filesize=30.2 K 2024-11-20T10:24:42,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/bd98e428b9374a3dbc7610e7ecf45741 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/bd98e428b9374a3dbc7610e7ecf45741 2024-11-20T10:24:42,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098342092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098342092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,111 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/bd98e428b9374a3dbc7610e7ecf45741, entries=150, sequenceid=41, filesize=11.7 K 2024-11-20T10:24:42,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/9d9fedfda1164f8781070cca9eb8eca5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/9d9fedfda1164f8781070cca9eb8eca5 2024-11-20T10:24:42,121 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/9d9fedfda1164f8781070cca9eb8eca5, entries=150, sequenceid=41, filesize=11.7 K 2024-11-20T10:24:42,122 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for a4b00f80e7a706f41be54575cc7fbb32 in 152ms, sequenceid=41, compaction requested=false 2024-11-20T10:24:42,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:42,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:42,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-20T10:24:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-20T10:24:42,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-20T10:24:42,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 469 msec 2024-11-20T10:24:42,134 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 482 msec 2024-11-20T10:24:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:42,199 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:42,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-20T10:24:42,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120eaaca37c1c02443bbf68a8bae18803a6_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098282088/Put/seqid=0 2024-11-20T10:24:42,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741962_1138 (size=17034) 2024-11-20T10:24:42,226 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:42,231 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120eaaca37c1c02443bbf68a8bae18803a6_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120eaaca37c1c02443bbf68a8bae18803a6_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:42,232 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/4ef62091e21647c589e01249fa790249, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:42,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/4ef62091e21647c589e01249fa790249 is 175, key is test_row_0/A:col10/1732098282088/Put/seqid=0 2024-11-20T10:24:42,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741963_1139 (size=48139) 2024-11-20T10:24:42,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098342241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098342244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098342246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098342250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098342250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T10:24:42,277 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:42,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:42,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-20T10:24:42,279 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:42,281 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:42,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:42,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T10:24:42,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098342356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098342356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098342357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098342367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098342367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T10:24:42,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T10:24:42,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:42,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:42,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:42,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:42,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:42,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098342567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098342567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098342567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098342577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098342577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T10:24:42,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:42,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:42,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:42,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:42,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T10:24:42,643 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/4ef62091e21647c589e01249fa790249 2024-11-20T10:24:42,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/8aeea50801bb496a927d448bdcdfce7a is 50, key is test_row_0/B:col10/1732098282088/Put/seqid=0 2024-11-20T10:24:42,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741964_1140 (size=12001) 2024-11-20T10:24:42,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/8aeea50801bb496a927d448bdcdfce7a 2024-11-20T10:24:42,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/90fa0e76f2ff4f64b27fa17da42c59b6 is 50, key is test_row_0/C:col10/1732098282088/Put/seqid=0 2024-11-20T10:24:42,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741965_1141 (size=12001) 2024-11-20T10:24:42,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T10:24:42,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:42,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:42,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:42,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:42,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:42,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098342877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098342877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098342878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098342887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098342887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:42,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T10:24:42,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:42,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:42,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:42,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:42,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T10:24:43,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T10:24:43,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:43,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:43,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:43,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:43,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:43,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/90fa0e76f2ff4f64b27fa17da42c59b6 2024-11-20T10:24:43,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/4ef62091e21647c589e01249fa790249 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/4ef62091e21647c589e01249fa790249 2024-11-20T10:24:43,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/4ef62091e21647c589e01249fa790249, entries=250, sequenceid=56, filesize=47.0 K 2024-11-20T10:24:43,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/8aeea50801bb496a927d448bdcdfce7a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8aeea50801bb496a927d448bdcdfce7a 2024-11-20T10:24:43,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8aeea50801bb496a927d448bdcdfce7a, entries=150, sequenceid=56, filesize=11.7 K 2024-11-20T10:24:43,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/90fa0e76f2ff4f64b27fa17da42c59b6 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/90fa0e76f2ff4f64b27fa17da42c59b6 2024-11-20T10:24:43,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/90fa0e76f2ff4f64b27fa17da42c59b6, entries=150, sequenceid=56, filesize=11.7 K 2024-11-20T10:24:43,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for a4b00f80e7a706f41be54575cc7fbb32 in 935ms, sequenceid=56, compaction requested=true 2024-11-20T10:24:43,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:43,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:43,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:43,134 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:43,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:43,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:43,134 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:43,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:43,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:43,136 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:43,136 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:43,136 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/B is initiating minor compaction (all files) 2024-11-20T10:24:43,136 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/A is initiating minor compaction (all files) 2024-11-20T10:24:43,136 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/B in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:43,136 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/A in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:43,136 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f1e0e20571cb4b6f86c3223a23b3cfb4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/bd98e428b9374a3dbc7610e7ecf45741, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8aeea50801bb496a927d448bdcdfce7a] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=35.2 K 2024-11-20T10:24:43,136 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/aa80b20109c744609b84e7eb6a49ca4f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/1321e7484d9e4f8caadb491ead79c432, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/4ef62091e21647c589e01249fa790249] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=107.5 K 2024-11-20T10:24:43,136 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:43,136 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/aa80b20109c744609b84e7eb6a49ca4f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/1321e7484d9e4f8caadb491ead79c432, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/4ef62091e21647c589e01249fa790249] 2024-11-20T10:24:43,137 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f1e0e20571cb4b6f86c3223a23b3cfb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732098281672 2024-11-20T10:24:43,137 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa80b20109c744609b84e7eb6a49ca4f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732098281672 2024-11-20T10:24:43,137 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting bd98e428b9374a3dbc7610e7ecf45741, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732098281712 2024-11-20T10:24:43,137 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1321e7484d9e4f8caadb491ead79c432, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732098281712 2024-11-20T10:24:43,138 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aeea50801bb496a927d448bdcdfce7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732098282076 2024-11-20T10:24:43,138 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ef62091e21647c589e01249fa790249, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732098282076 2024-11-20T10:24:43,148 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#B#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:43,149 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:43,149 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/2487551c3c4340b8b7f21d790c71e92b is 50, key is test_row_0/B:col10/1732098282088/Put/seqid=0 2024-11-20T10:24:43,153 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112068d26026c6d04cc687b20acbb4969e14_a4b00f80e7a706f41be54575cc7fbb32 store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:43,160 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112068d26026c6d04cc687b20acbb4969e14_a4b00f80e7a706f41be54575cc7fbb32, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:43,160 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112068d26026c6d04cc687b20acbb4969e14_a4b00f80e7a706f41be54575cc7fbb32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:43,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741966_1142 (size=12104) 2024-11-20T10:24:43,191 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/2487551c3c4340b8b7f21d790c71e92b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/2487551c3c4340b8b7f21d790c71e92b 2024-11-20T10:24:43,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741967_1143 (size=4469) 2024-11-20T10:24:43,202 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/B of a4b00f80e7a706f41be54575cc7fbb32 into 2487551c3c4340b8b7f21d790c71e92b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:43,202 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:43,202 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/B, priority=13, startTime=1732098283134; duration=0sec 2024-11-20T10:24:43,202 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:43,202 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:B 2024-11-20T10:24:43,202 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:43,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T10:24:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:43,203 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-20T10:24:43,205 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:43,205 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/C is initiating minor compaction (all files) 2024-11-20T10:24:43,205 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/C in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:43,206 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/350c4f9f759d49ff880320fe83915e16, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/9d9fedfda1164f8781070cca9eb8eca5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/90fa0e76f2ff4f64b27fa17da42c59b6] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=35.2 K 2024-11-20T10:24:43,206 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 350c4f9f759d49ff880320fe83915e16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732098281672 2024-11-20T10:24:43,207 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d9fedfda1164f8781070cca9eb8eca5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732098281712 2024-11-20T10:24:43,207 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 90fa0e76f2ff4f64b27fa17da42c59b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732098282076 2024-11-20T10:24:43,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f0917a5087aa436380e927a50d8bd957_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098282246/Put/seqid=0 2024-11-20T10:24:43,230 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#C#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:43,231 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/417e37301aed406297c2d72f120ed849 is 50, key is test_row_0/C:col10/1732098282088/Put/seqid=0 2024-11-20T10:24:43,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741968_1144 (size=12154) 2024-11-20T10:24:43,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:43,242 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f0917a5087aa436380e927a50d8bd957_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f0917a5087aa436380e927a50d8bd957_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:43,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/f7f66625a52641f7abcb02c708c895a9, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:43,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/f7f66625a52641f7abcb02c708c895a9 is 175, key is test_row_0/A:col10/1732098282246/Put/seqid=0 2024-11-20T10:24:43,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741969_1145 (size=12104) 2024-11-20T10:24:43,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741970_1146 (size=30955) 2024-11-20T10:24:43,339 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T10:24:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:43,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:43,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098343403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:43,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098343404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:43,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098343406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:43,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:43,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098343407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:43,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:43,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098343409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T10:24:43,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098343517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:43,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:43,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098343517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:43,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:43,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098343517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:43,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:43,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:43,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098343517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098343517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:43,603 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#A#compaction#124 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:43,603 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/fec1e71d117941979637b68123bfecd3 is 175, key is test_row_0/A:col10/1732098282088/Put/seqid=0 2024-11-20T10:24:43,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741971_1147 (size=31058) 2024-11-20T10:24:43,660 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/417e37301aed406297c2d72f120ed849 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/417e37301aed406297c2d72f120ed849 2024-11-20T10:24:43,661 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/f7f66625a52641f7abcb02c708c895a9 2024-11-20T10:24:43,668 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/C of a4b00f80e7a706f41be54575cc7fbb32 into 417e37301aed406297c2d72f120ed849(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:43,668 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:43,669 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/C, priority=13, startTime=1732098283135; duration=0sec 2024-11-20T10:24:43,669 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:43,669 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:C 2024-11-20T10:24:43,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/93e742d915374c60839b9154174ddfdb is 50, key is test_row_0/B:col10/1732098282246/Put/seqid=0 2024-11-20T10:24:43,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741972_1148 (size=12001) 2024-11-20T10:24:43,681 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/93e742d915374c60839b9154174ddfdb 2024-11-20T10:24:43,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/8e2f53eb513c4529a0aee4381ac96585 is 50, key is test_row_0/C:col10/1732098282246/Put/seqid=0 2024-11-20T10:24:43,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741973_1149 (size=12001) 2024-11-20T10:24:43,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:43,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098343727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098343727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098343727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:43,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098343727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098343728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:43,841 INFO [master/8c533afd331c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-20T10:24:43,841 INFO [master/8c533afd331c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-20T10:24:44,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098344036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098344036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098344037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098344036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098344037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,071 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/fec1e71d117941979637b68123bfecd3 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/fec1e71d117941979637b68123bfecd3 2024-11-20T10:24:44,076 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/A of a4b00f80e7a706f41be54575cc7fbb32 into fec1e71d117941979637b68123bfecd3(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:44,076 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:44,076 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/A, priority=13, startTime=1732098283134; duration=0sec 2024-11-20T10:24:44,076 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:44,076 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:A 2024-11-20T10:24:44,110 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/8e2f53eb513c4529a0aee4381ac96585 2024-11-20T10:24:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/f7f66625a52641f7abcb02c708c895a9 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/f7f66625a52641f7abcb02c708c895a9 2024-11-20T10:24:44,122 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/f7f66625a52641f7abcb02c708c895a9, entries=150, sequenceid=79, filesize=30.2 K 2024-11-20T10:24:44,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/93e742d915374c60839b9154174ddfdb as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/93e742d915374c60839b9154174ddfdb 2024-11-20T10:24:44,129 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/93e742d915374c60839b9154174ddfdb, entries=150, sequenceid=79, filesize=11.7 K 2024-11-20T10:24:44,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/8e2f53eb513c4529a0aee4381ac96585 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/8e2f53eb513c4529a0aee4381ac96585 2024-11-20T10:24:44,136 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/8e2f53eb513c4529a0aee4381ac96585, entries=150, sequenceid=79, filesize=11.7 K 2024-11-20T10:24:44,137 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for a4b00f80e7a706f41be54575cc7fbb32 in 934ms, sequenceid=79, compaction requested=false 2024-11-20T10:24:44,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:44,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:44,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-20T10:24:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-20T10:24:44,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-20T10:24:44,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8580 sec 2024-11-20T10:24:44,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.8640 sec 2024-11-20T10:24:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T10:24:44,427 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:44,428 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-20T10:24:44,429 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T10:24:44,433 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:44,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T10:24:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:44,551 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:44,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T10:24:44,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120733530a295ed497bbe5e2a0b57023767_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098283406/Put/seqid=0 2024-11-20T10:24:44,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741974_1150 (size=17034) 2024-11-20T10:24:44,570 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:44,575 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120733530a295ed497bbe5e2a0b57023767_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120733530a295ed497bbe5e2a0b57023767_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:44,577 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/7482635e304143d7af513e4241abdb05, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:44,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/7482635e304143d7af513e4241abdb05 is 175, key is test_row_0/A:col10/1732098283406/Put/seqid=0 2024-11-20T10:24:44,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098344578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T10:24:44,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:44,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:44,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:44,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:44,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:44,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741975_1151 (size=48139) 2024-11-20T10:24:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:44,588 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/7482635e304143d7af513e4241abdb05 2024-11-20T10:24:44,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098344579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098344581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098344582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098344583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/8bfeb86640754b6b8cccdc18faabb5bc is 50, key is test_row_0/B:col10/1732098283406/Put/seqid=0 2024-11-20T10:24:44,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741976_1152 (size=12001) 2024-11-20T10:24:44,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098344687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098344696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098344697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098344697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098344697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T10:24:44,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:44,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:44,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:44,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:44,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T10:24:44,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T10:24:44,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:44,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:44,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:44,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:44,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:44,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098344896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098344907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098344917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098344917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:44,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098344917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:45,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/8bfeb86640754b6b8cccdc18faabb5bc 2024-11-20T10:24:45,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/e3c04151ecb54b62b0b2608d63d6024d is 50, key is test_row_0/C:col10/1732098283406/Put/seqid=0 2024-11-20T10:24:45,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741977_1153 (size=12001) 2024-11-20T10:24:45,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T10:24:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:45,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:45,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:45,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:45,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:45,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:45,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T10:24:45,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T10:24:45,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:45,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:45,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:45,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:45,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:45,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:45,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098345207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:45,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:45,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098345217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:45,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:45,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:45,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098345226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:45,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098345226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:45,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098345227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:45,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T10:24:45,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:45,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:45,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:45,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:45,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:45,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/e3c04151ecb54b62b0b2608d63d6024d 2024-11-20T10:24:45,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/7482635e304143d7af513e4241abdb05 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/7482635e304143d7af513e4241abdb05 2024-11-20T10:24:45,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/7482635e304143d7af513e4241abdb05, entries=250, sequenceid=98, filesize=47.0 K 2024-11-20T10:24:45,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/8bfeb86640754b6b8cccdc18faabb5bc as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8bfeb86640754b6b8cccdc18faabb5bc 2024-11-20T10:24:45,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8bfeb86640754b6b8cccdc18faabb5bc, entries=150, sequenceid=98, filesize=11.7 K 2024-11-20T10:24:45,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/e3c04151ecb54b62b0b2608d63d6024d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/e3c04151ecb54b62b0b2608d63d6024d 2024-11-20T10:24:45,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/e3c04151ecb54b62b0b2608d63d6024d, entries=150, sequenceid=98, filesize=11.7 K 2024-11-20T10:24:45,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a4b00f80e7a706f41be54575cc7fbb32 in 915ms, sequenceid=98, compaction requested=true 2024-11-20T10:24:45,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:45,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:45,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:45,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:45,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T10:24:45,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:45,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T10:24:45,467 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:45,467 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:45,469 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:45,469 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/C is initiating minor compaction (all files) 2024-11-20T10:24:45,469 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/C in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:45,469 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/417e37301aed406297c2d72f120ed849, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/8e2f53eb513c4529a0aee4381ac96585, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/e3c04151ecb54b62b0b2608d63d6024d] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=35.3 K 2024-11-20T10:24:45,469 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:45,469 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/A is initiating minor compaction (all files) 2024-11-20T10:24:45,469 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/A in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:45,470 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/fec1e71d117941979637b68123bfecd3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/f7f66625a52641f7abcb02c708c895a9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/7482635e304143d7af513e4241abdb05] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=107.6 K 2024-11-20T10:24:45,470 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 417e37301aed406297c2d72f120ed849, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732098282076 2024-11-20T10:24:45,470 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:45,470 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/fec1e71d117941979637b68123bfecd3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/f7f66625a52641f7abcb02c708c895a9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/7482635e304143d7af513e4241abdb05] 2024-11-20T10:24:45,470 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e2f53eb513c4529a0aee4381ac96585, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732098282238 2024-11-20T10:24:45,470 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting fec1e71d117941979637b68123bfecd3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732098282076 2024-11-20T10:24:45,470 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting e3c04151ecb54b62b0b2608d63d6024d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732098283406 2024-11-20T10:24:45,471 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7f66625a52641f7abcb02c708c895a9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732098282238 2024-11-20T10:24:45,472 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7482635e304143d7af513e4241abdb05, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732098283404 2024-11-20T10:24:45,481 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:45,482 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#C#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:45,482 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/98f847fa9a2a4c7db28eaebd13848d9e is 50, key is test_row_0/C:col10/1732098283406/Put/seqid=0 2024-11-20T10:24:45,490 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112011080d79010042a69be48ebbaccddc79_a4b00f80e7a706f41be54575cc7fbb32 store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:45,495 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112011080d79010042a69be48ebbaccddc79_a4b00f80e7a706f41be54575cc7fbb32, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:45,495 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112011080d79010042a69be48ebbaccddc79_a4b00f80e7a706f41be54575cc7fbb32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:45,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T10:24:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:45,506 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T10:24:45,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741978_1154 (size=12207) 2024-11-20T10:24:45,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200370858ba4354139af58d2cdbde7a2b9_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098284574/Put/seqid=0 2024-11-20T10:24:45,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741979_1155 (size=4469) 2024-11-20T10:24:45,549 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#A#compaction#133 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:45,549 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/98f847fa9a2a4c7db28eaebd13848d9e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/98f847fa9a2a4c7db28eaebd13848d9e 2024-11-20T10:24:45,549 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/02f4c4fb91c542a2ad2a5eb9a64a3bdf is 175, key is test_row_0/A:col10/1732098283406/Put/seqid=0 2024-11-20T10:24:45,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741980_1156 (size=12154) 2024-11-20T10:24:45,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:45,558 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/C of a4b00f80e7a706f41be54575cc7fbb32 into 98f847fa9a2a4c7db28eaebd13848d9e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:45,558 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:45,558 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/C, priority=13, startTime=1732098285467; duration=0sec 2024-11-20T10:24:45,558 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:45,558 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:C 2024-11-20T10:24:45,558 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:45,560 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:45,560 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/B is initiating minor compaction (all files) 2024-11-20T10:24:45,560 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/B in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:45,560 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/2487551c3c4340b8b7f21d790c71e92b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/93e742d915374c60839b9154174ddfdb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8bfeb86640754b6b8cccdc18faabb5bc] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=35.3 K 2024-11-20T10:24:45,564 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200370858ba4354139af58d2cdbde7a2b9_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200370858ba4354139af58d2cdbde7a2b9_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:45,565 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2487551c3c4340b8b7f21d790c71e92b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732098282076 2024-11-20T10:24:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/f219e7e106564862b963f73952e6a9d7, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:45,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/f219e7e106564862b963f73952e6a9d7 is 175, key is test_row_0/A:col10/1732098284574/Put/seqid=0 2024-11-20T10:24:45,567 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 93e742d915374c60839b9154174ddfdb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732098282238 2024-11-20T10:24:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T10:24:45,571 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bfeb86640754b6b8cccdc18faabb5bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732098283406 2024-11-20T10:24:45,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741981_1157 (size=31161) 2024-11-20T10:24:45,594 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#B#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:45,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741982_1158 (size=30955) 2024-11-20T10:24:45,595 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/523f065e12054fa18659606499316cb0 is 50, key is test_row_0/B:col10/1732098283406/Put/seqid=0 2024-11-20T10:24:45,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741983_1159 (size=12207) 2024-11-20T10:24:45,601 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/f219e7e106564862b963f73952e6a9d7 2024-11-20T10:24:45,606 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/523f065e12054fa18659606499316cb0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/523f065e12054fa18659606499316cb0 2024-11-20T10:24:45,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/c2c927ae70974d829be0bd2679d3a441 is 50, key is test_row_0/B:col10/1732098284574/Put/seqid=0 2024-11-20T10:24:45,613 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/B of a4b00f80e7a706f41be54575cc7fbb32 into 523f065e12054fa18659606499316cb0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:45,613 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:45,613 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/B, priority=13, startTime=1732098285467; duration=0sec 2024-11-20T10:24:45,613 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:45,613 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:B 2024-11-20T10:24:45,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741984_1160 (size=12001) 2024-11-20T10:24:45,627 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/c2c927ae70974d829be0bd2679d3a441 2024-11-20T10:24:45,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/95cd41b3a181402d887c329fb97aa974 is 50, key is test_row_0/C:col10/1732098284574/Put/seqid=0 2024-11-20T10:24:45,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741985_1161 (size=12001) 2024-11-20T10:24:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:45,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:45,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098345745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:45,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098345746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:45,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098345747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098345748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:45,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098345748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:45,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:45,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098345857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098345857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:45,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:45,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098345857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:45,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098345857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098345857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,000 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/02f4c4fb91c542a2ad2a5eb9a64a3bdf as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/02f4c4fb91c542a2ad2a5eb9a64a3bdf 2024-11-20T10:24:46,012 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/A of a4b00f80e7a706f41be54575cc7fbb32 into 02f4c4fb91c542a2ad2a5eb9a64a3bdf(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:46,013 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:46,013 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/A, priority=13, startTime=1732098285466; duration=0sec 2024-11-20T10:24:46,013 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:46,013 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:A 2024-11-20T10:24:46,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098346067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098346067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098346067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098346067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098346068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,095 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/95cd41b3a181402d887c329fb97aa974 2024-11-20T10:24:46,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/f219e7e106564862b963f73952e6a9d7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/f219e7e106564862b963f73952e6a9d7 2024-11-20T10:24:46,105 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/f219e7e106564862b963f73952e6a9d7, entries=150, sequenceid=117, filesize=30.2 K 2024-11-20T10:24:46,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/c2c927ae70974d829be0bd2679d3a441 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/c2c927ae70974d829be0bd2679d3a441 2024-11-20T10:24:46,112 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/c2c927ae70974d829be0bd2679d3a441, entries=150, sequenceid=117, filesize=11.7 K 2024-11-20T10:24:46,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/95cd41b3a181402d887c329fb97aa974 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/95cd41b3a181402d887c329fb97aa974 2024-11-20T10:24:46,118 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/95cd41b3a181402d887c329fb97aa974, entries=150, sequenceid=117, filesize=11.7 K 2024-11-20T10:24:46,119 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for a4b00f80e7a706f41be54575cc7fbb32 in 613ms, sequenceid=117, compaction requested=false 2024-11-20T10:24:46,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:46,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:46,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-20T10:24:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-20T10:24:46,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-20T10:24:46,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6870 sec 2024-11-20T10:24:46,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.6950 sec 2024-11-20T10:24:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:46,379 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:46,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T10:24:46,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209593962452ea4dca8721af494a1d3b02_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098285746/Put/seqid=0 2024-11-20T10:24:46,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741986_1162 (size=14794) 2024-11-20T10:24:46,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098346393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098346397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098346397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098346398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098346399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098346508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098346508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098346510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098346510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098346510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T10:24:46,587 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:46,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-20T10:24:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T10:24:46,593 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:46,594 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:46,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T10:24:46,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098346717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098346717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098346717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098346717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098346717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:46,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T10:24:46,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:46,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:46,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:46,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:46,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:46,796 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:46,800 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209593962452ea4dca8721af494a1d3b02_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209593962452ea4dca8721af494a1d3b02_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:46,801 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/5e6bb6bcc9cf4801bb3b081a80b3d20e, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:46,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/5e6bb6bcc9cf4801bb3b081a80b3d20e is 175, key is test_row_0/A:col10/1732098285746/Put/seqid=0 2024-11-20T10:24:46,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741987_1163 (size=39749) 2024-11-20T10:24:46,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T10:24:46,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:46,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:46,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:46,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:46,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T10:24:47,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098347027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:47,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098347027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098347027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098347027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098347028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:47,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T10:24:47,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:47,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:47,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:47,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:47,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:47,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:47,206 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/5e6bb6bcc9cf4801bb3b081a80b3d20e 2024-11-20T10:24:47,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T10:24:47,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:47,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:47,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:47,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:47,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:47,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/8adc118a372c4a46bbbf187dc8a58427 is 50, key is test_row_0/B:col10/1732098285746/Put/seqid=0 2024-11-20T10:24:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T10:24:47,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741988_1164 (size=12151) 2024-11-20T10:24:47,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T10:24:47,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:47,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:47,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:47,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:47,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:47,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T10:24:47,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:47,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:47,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:47,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:47,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:47,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098347536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:47,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:47,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098347537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098347537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098347537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098347537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:47,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/8adc118a372c4a46bbbf187dc8a58427 2024-11-20T10:24:47,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/88e83c66c1d748ecbfaed40cd77dcfb4 is 50, key is test_row_0/C:col10/1732098285746/Put/seqid=0 2024-11-20T10:24:47,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741989_1165 (size=12151) 2024-11-20T10:24:47,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/88e83c66c1d748ecbfaed40cd77dcfb4 2024-11-20T10:24:47,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/5e6bb6bcc9cf4801bb3b081a80b3d20e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/5e6bb6bcc9cf4801bb3b081a80b3d20e 2024-11-20T10:24:47,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/5e6bb6bcc9cf4801bb3b081a80b3d20e, entries=200, sequenceid=139, filesize=38.8 K 2024-11-20T10:24:47,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/8adc118a372c4a46bbbf187dc8a58427 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8adc118a372c4a46bbbf187dc8a58427 2024-11-20T10:24:47,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8adc118a372c4a46bbbf187dc8a58427, entries=150, sequenceid=139, filesize=11.9 K 2024-11-20T10:24:47,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/88e83c66c1d748ecbfaed40cd77dcfb4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/88e83c66c1d748ecbfaed40cd77dcfb4 2024-11-20T10:24:47,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/88e83c66c1d748ecbfaed40cd77dcfb4, entries=150, sequenceid=139, filesize=11.9 K 2024-11-20T10:24:47,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for a4b00f80e7a706f41be54575cc7fbb32 in 1279ms, sequenceid=139, compaction requested=true 2024-11-20T10:24:47,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:47,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:47,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:47,658 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:47,658 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:47,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:47,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:47,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:47,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:47,659 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:47,659 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:47,660 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/A is initiating minor compaction (all files) 2024-11-20T10:24:47,660 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/B is initiating minor compaction (all files) 2024-11-20T10:24:47,660 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/A in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:47,660 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/B in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:47,660 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/02f4c4fb91c542a2ad2a5eb9a64a3bdf, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/f219e7e106564862b963f73952e6a9d7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/5e6bb6bcc9cf4801bb3b081a80b3d20e] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=99.5 K 2024-11-20T10:24:47,660 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/523f065e12054fa18659606499316cb0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/c2c927ae70974d829be0bd2679d3a441, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8adc118a372c4a46bbbf187dc8a58427] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=35.5 K 2024-11-20T10:24:47,660 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:47,660 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/02f4c4fb91c542a2ad2a5eb9a64a3bdf, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/f219e7e106564862b963f73952e6a9d7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/5e6bb6bcc9cf4801bb3b081a80b3d20e] 2024-11-20T10:24:47,660 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02f4c4fb91c542a2ad2a5eb9a64a3bdf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732098283406 2024-11-20T10:24:47,660 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 523f065e12054fa18659606499316cb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732098283406 2024-11-20T10:24:47,661 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c2c927ae70974d829be0bd2679d3a441, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732098284570 2024-11-20T10:24:47,661 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f219e7e106564862b963f73952e6a9d7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732098284570 2024-11-20T10:24:47,661 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8adc118a372c4a46bbbf187dc8a58427, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732098285746 2024-11-20T10:24:47,661 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e6bb6bcc9cf4801bb3b081a80b3d20e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732098285746 2024-11-20T10:24:47,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T10:24:47,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:47,670 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T10:24:47,671 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:47,674 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#B#compaction#142 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:47,674 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/06ea10a5d579462a941d4388396a5c39 is 50, key is test_row_0/B:col10/1732098285746/Put/seqid=0 2024-11-20T10:24:47,681 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120d6b7b76ebf8d4b7ebd840c08385a39e3_a4b00f80e7a706f41be54575cc7fbb32 store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:47,684 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120d6b7b76ebf8d4b7ebd840c08385a39e3_a4b00f80e7a706f41be54575cc7fbb32, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:47,684 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d6b7b76ebf8d4b7ebd840c08385a39e3_a4b00f80e7a706f41be54575cc7fbb32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:47,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741990_1166 (size=12459) 2024-11-20T10:24:47,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ba7021a9e1914954b20818b72db1ae77_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098286397/Put/seqid=0 2024-11-20T10:24:47,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741992_1168 (size=12304) 2024-11-20T10:24:47,705 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/06ea10a5d579462a941d4388396a5c39 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/06ea10a5d579462a941d4388396a5c39 2024-11-20T10:24:47,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741991_1167 (size=4469) 2024-11-20T10:24:47,713 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/B of a4b00f80e7a706f41be54575cc7fbb32 into 06ea10a5d579462a941d4388396a5c39(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:47,713 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:47,713 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/B, priority=13, startTime=1732098287658; duration=0sec 2024-11-20T10:24:47,713 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:47,713 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:B 2024-11-20T10:24:47,713 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:47,715 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:47,716 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/C is initiating minor compaction (all files) 2024-11-20T10:24:47,716 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/C in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:47,716 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/98f847fa9a2a4c7db28eaebd13848d9e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/95cd41b3a181402d887c329fb97aa974, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/88e83c66c1d748ecbfaed40cd77dcfb4] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=35.5 K 2024-11-20T10:24:47,716 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 98f847fa9a2a4c7db28eaebd13848d9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732098283406 2024-11-20T10:24:47,717 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 95cd41b3a181402d887c329fb97aa974, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732098284570 2024-11-20T10:24:47,717 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 88e83c66c1d748ecbfaed40cd77dcfb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732098285746 2024-11-20T10:24:47,724 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#C#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:47,725 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/3a3799442c8248658ea6243be2d2b8cd is 50, key is test_row_0/C:col10/1732098285746/Put/seqid=0 2024-11-20T10:24:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T10:24:47,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741993_1169 (size=12459) 2024-11-20T10:24:47,887 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T10:24:47,887 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T10:24:48,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:48,109 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#A#compaction#141 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:48,110 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/b6c2fcd8868b4e36baa68e9d2d7827e9 is 175, key is test_row_0/A:col10/1732098285746/Put/seqid=0 2024-11-20T10:24:48,112 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ba7021a9e1914954b20818b72db1ae77_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba7021a9e1914954b20818b72db1ae77_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:48,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/4b79545a45d948fb9d321e722a631669, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:48,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/4b79545a45d948fb9d321e722a631669 is 175, key is test_row_0/A:col10/1732098286397/Put/seqid=0 2024-11-20T10:24:48,140 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/3a3799442c8248658ea6243be2d2b8cd as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/3a3799442c8248658ea6243be2d2b8cd 2024-11-20T10:24:48,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741994_1170 (size=31413) 2024-11-20T10:24:48,151 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/C of a4b00f80e7a706f41be54575cc7fbb32 into 3a3799442c8248658ea6243be2d2b8cd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:48,151 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:48,151 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/C, priority=13, startTime=1732098287659; duration=0sec 2024-11-20T10:24:48,152 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:48,152 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:C 2024-11-20T10:24:48,152 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/b6c2fcd8868b4e36baa68e9d2d7827e9 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/b6c2fcd8868b4e36baa68e9d2d7827e9 2024-11-20T10:24:48,160 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/A of a4b00f80e7a706f41be54575cc7fbb32 into b6c2fcd8868b4e36baa68e9d2d7827e9(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:48,160 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:48,160 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/A, priority=13, startTime=1732098287658; duration=0sec 2024-11-20T10:24:48,160 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:48,160 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:A 2024-11-20T10:24:48,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741995_1171 (size=31105) 2024-11-20T10:24:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:48,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:48,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098348561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:48,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:48,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098348564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:48,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:48,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098348565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:48,569 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/4b79545a45d948fb9d321e722a631669 2024-11-20T10:24:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:48,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098348566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:48,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098348567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:48,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/5cf2df5535474364b4df50f2defdcf8e is 50, key is test_row_0/B:col10/1732098286397/Put/seqid=0 2024-11-20T10:24:48,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741996_1172 (size=12151) 2024-11-20T10:24:48,586 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/5cf2df5535474364b4df50f2defdcf8e 2024-11-20T10:24:48,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/554b3d5c95fe493784cbc67767b889f5 is 50, key is test_row_0/C:col10/1732098286397/Put/seqid=0 2024-11-20T10:24:48,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741997_1173 (size=12151) 2024-11-20T10:24:48,600 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/554b3d5c95fe493784cbc67767b889f5 2024-11-20T10:24:48,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/4b79545a45d948fb9d321e722a631669 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/4b79545a45d948fb9d321e722a631669 2024-11-20T10:24:48,609 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/4b79545a45d948fb9d321e722a631669, entries=150, sequenceid=156, filesize=30.4 K 2024-11-20T10:24:48,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/5cf2df5535474364b4df50f2defdcf8e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/5cf2df5535474364b4df50f2defdcf8e 2024-11-20T10:24:48,616 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/5cf2df5535474364b4df50f2defdcf8e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T10:24:48,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/554b3d5c95fe493784cbc67767b889f5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/554b3d5c95fe493784cbc67767b889f5 2024-11-20T10:24:48,621 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/554b3d5c95fe493784cbc67767b889f5, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T10:24:48,622 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a4b00f80e7a706f41be54575cc7fbb32 in 953ms, sequenceid=156, compaction requested=false 2024-11-20T10:24:48,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:48,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:48,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-20T10:24:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-20T10:24:48,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-20T10:24:48,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0300 sec 2024-11-20T10:24:48,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.0390 sec 2024-11-20T10:24:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:48,679 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:48,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T10:24:48,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ba2fbce39f0041459e910f9c3e864e39_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098288679/Put/seqid=0 2024-11-20T10:24:48,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741998_1174 (size=17284) 2024-11-20T10:24:48,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098348695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:48,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098348696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:48,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:48,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:48,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098348698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:48,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:48,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098348699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:48,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098348699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T10:24:48,747 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:48,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-20T10:24:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T10:24:48,749 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:48,750 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:48,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:48,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098348807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:48,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098348807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098348807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098348807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098348807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:48,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T10:24:48,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T10:24:48,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:48,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:48,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:48,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:48,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:49,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098349016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098349016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:49,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098349017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098349017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098349017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:49,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T10:24:49,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:49,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:49,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:49,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T10:24:49,098 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:49,105 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ba2fbce39f0041459e910f9c3e864e39_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba2fbce39f0041459e910f9c3e864e39_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:49,106 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/35764acac8894c1c9ad6d752b24dfd1a, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:49,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/35764acac8894c1c9ad6d752b24dfd1a is 175, key is test_row_0/A:col10/1732098288679/Put/seqid=0 2024-11-20T10:24:49,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741999_1175 (size=48389) 2024-11-20T10:24:49,126 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/35764acac8894c1c9ad6d752b24dfd1a 2024-11-20T10:24:49,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/f16fa72f1fe94592b3cbdbfcb6408a16 is 50, key is test_row_0/B:col10/1732098288679/Put/seqid=0 2024-11-20T10:24:49,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742000_1176 (size=12151) 2024-11-20T10:24:49,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T10:24:49,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:49,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:49,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098349327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098349327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:49,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098349327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:49,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098349327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:49,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098349327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:49,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T10:24:49,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:49,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:49,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:49,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T10:24:49,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T10:24:49,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:49,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:49,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:49,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/f16fa72f1fe94592b3cbdbfcb6408a16 2024-11-20T10:24:49,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/56955b84639e47e5be9638d8283cec38 is 50, key is test_row_0/C:col10/1732098288679/Put/seqid=0 2024-11-20T10:24:49,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742001_1177 (size=12151) 2024-11-20T10:24:49,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T10:24:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:49,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T10:24:49,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:49,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:49,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:49,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098349837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098349837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:49,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098349837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:49,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098349837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:49,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098349837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T10:24:49,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T10:24:49,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:49,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:49,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:49,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:50,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/56955b84639e47e5be9638d8283cec38 2024-11-20T10:24:50,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/35764acac8894c1c9ad6d752b24dfd1a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/35764acac8894c1c9ad6d752b24dfd1a 2024-11-20T10:24:50,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/35764acac8894c1c9ad6d752b24dfd1a, entries=250, sequenceid=180, filesize=47.3 K 2024-11-20T10:24:50,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/f16fa72f1fe94592b3cbdbfcb6408a16 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f16fa72f1fe94592b3cbdbfcb6408a16 2024-11-20T10:24:50,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f16fa72f1fe94592b3cbdbfcb6408a16, entries=150, sequenceid=180, filesize=11.9 K 2024-11-20T10:24:50,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/56955b84639e47e5be9638d8283cec38 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/56955b84639e47e5be9638d8283cec38 2024-11-20T10:24:50,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/56955b84639e47e5be9638d8283cec38, entries=150, sequenceid=180, filesize=11.9 K 2024-11-20T10:24:50,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for a4b00f80e7a706f41be54575cc7fbb32 in 1367ms, sequenceid=180, compaction requested=true 2024-11-20T10:24:50,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:50,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:50,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:50,047 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:50,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:50,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:50,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:50,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T10:24:50,047 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:50,048 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:50,048 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:50,048 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/B is initiating minor compaction (all files) 2024-11-20T10:24:50,048 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/A is initiating minor compaction (all files) 2024-11-20T10:24:50,048 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/A in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:50,048 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/b6c2fcd8868b4e36baa68e9d2d7827e9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/4b79545a45d948fb9d321e722a631669, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/35764acac8894c1c9ad6d752b24dfd1a] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=108.3 K 2024-11-20T10:24:50,048 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:50,048 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/b6c2fcd8868b4e36baa68e9d2d7827e9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/4b79545a45d948fb9d321e722a631669, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/35764acac8894c1c9ad6d752b24dfd1a] 2024-11-20T10:24:50,048 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/B in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:50,049 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/06ea10a5d579462a941d4388396a5c39, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/5cf2df5535474364b4df50f2defdcf8e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f16fa72f1fe94592b3cbdbfcb6408a16] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=35.9 K 2024-11-20T10:24:50,049 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6c2fcd8868b4e36baa68e9d2d7827e9, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732098285746 2024-11-20T10:24:50,049 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 06ea10a5d579462a941d4388396a5c39, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732098285746 2024-11-20T10:24:50,049 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b79545a45d948fb9d321e722a631669, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732098286395 2024-11-20T10:24:50,050 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf2df5535474364b4df50f2defdcf8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732098286395 2024-11-20T10:24:50,050 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35764acac8894c1c9ad6d752b24dfd1a, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732098288558 2024-11-20T10:24:50,050 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f16fa72f1fe94592b3cbdbfcb6408a16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732098288565 2024-11-20T10:24:50,057 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:50,058 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#B#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:50,058 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/e45f3071b1f845fa899422a20e9ca08e is 50, key is test_row_0/B:col10/1732098288679/Put/seqid=0 2024-11-20T10:24:50,061 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120a40ee7429a334933bbfa19b554626414_a4b00f80e7a706f41be54575cc7fbb32 store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:50,063 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120a40ee7429a334933bbfa19b554626414_a4b00f80e7a706f41be54575cc7fbb32, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:50,063 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a40ee7429a334933bbfa19b554626414_a4b00f80e7a706f41be54575cc7fbb32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:50,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742002_1178 (size=12561) 2024-11-20T10:24:50,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742003_1179 (size=4469) 2024-11-20T10:24:50,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T10:24:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:50,146 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T10:24:50,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120684e57bbfbd44a23bd422ff7d6a0bb02_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098288697/Put/seqid=0 2024-11-20T10:24:50,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742004_1180 (size=12304) 2024-11-20T10:24:50,483 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/e45f3071b1f845fa899422a20e9ca08e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/e45f3071b1f845fa899422a20e9ca08e 2024-11-20T10:24:50,490 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/B of a4b00f80e7a706f41be54575cc7fbb32 into e45f3071b1f845fa899422a20e9ca08e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:50,490 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:50,490 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/B, priority=13, startTime=1732098290047; duration=0sec 2024-11-20T10:24:50,490 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:50,490 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:B 2024-11-20T10:24:50,490 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:50,493 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:50,493 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/C is initiating minor compaction (all files) 2024-11-20T10:24:50,493 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/C in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:50,493 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/3a3799442c8248658ea6243be2d2b8cd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/554b3d5c95fe493784cbc67767b889f5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/56955b84639e47e5be9638d8283cec38] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=35.9 K 2024-11-20T10:24:50,493 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a3799442c8248658ea6243be2d2b8cd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732098285746 2024-11-20T10:24:50,494 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 554b3d5c95fe493784cbc67767b889f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732098286395 2024-11-20T10:24:50,494 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 56955b84639e47e5be9638d8283cec38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732098288565 2024-11-20T10:24:50,496 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#A#compaction#151 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:50,497 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/e7e07c5d50954bdeadeb982f25fe29d8 is 175, key is test_row_0/A:col10/1732098288679/Put/seqid=0 2024-11-20T10:24:50,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742005_1181 (size=31515) 2024-11-20T10:24:50,514 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#C#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:50,515 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/fde646a1667a488e86b64bc6cbb92305 is 50, key is test_row_0/C:col10/1732098288679/Put/seqid=0 2024-11-20T10:24:50,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742006_1182 (size=12561) 2024-11-20T10:24:50,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,573 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120684e57bbfbd44a23bd422ff7d6a0bb02_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120684e57bbfbd44a23bd422ff7d6a0bb02_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:50,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/3a9f8bb1b3f54f0ea60f032e0a7fb1ea, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:50,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/3a9f8bb1b3f54f0ea60f032e0a7fb1ea is 175, key is test_row_0/A:col10/1732098288697/Put/seqid=0 2024-11-20T10:24:50,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742007_1183 (size=31105) 2024-11-20T10:24:50,587 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/3a9f8bb1b3f54f0ea60f032e0a7fb1ea 2024-11-20T10:24:50,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/d735cef392cd429caa5f21664f4b1fab is 50, key is test_row_0/B:col10/1732098288697/Put/seqid=0 2024-11-20T10:24:50,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742008_1184 (size=12151) 2024-11-20T10:24:50,622 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/d735cef392cd429caa5f21664f4b1fab 2024-11-20T10:24:50,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/63abb9b1b9a84e14896c784e4f85200e is 50, key is test_row_0/C:col10/1732098288697/Put/seqid=0 2024-11-20T10:24:50,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742009_1185 (size=12151) 2024-11-20T10:24:50,650 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/63abb9b1b9a84e14896c784e4f85200e 2024-11-20T10:24:50,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/3a9f8bb1b3f54f0ea60f032e0a7fb1ea as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/3a9f8bb1b3f54f0ea60f032e0a7fb1ea 2024-11-20T10:24:50,669 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/3a9f8bb1b3f54f0ea60f032e0a7fb1ea, entries=150, sequenceid=196, filesize=30.4 K 2024-11-20T10:24:50,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/d735cef392cd429caa5f21664f4b1fab as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/d735cef392cd429caa5f21664f4b1fab 2024-11-20T10:24:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,675 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/d735cef392cd429caa5f21664f4b1fab, entries=150, sequenceid=196, filesize=11.9 K 2024-11-20T10:24:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/63abb9b1b9a84e14896c784e4f85200e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/63abb9b1b9a84e14896c784e4f85200e 2024-11-20T10:24:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,683 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/63abb9b1b9a84e14896c784e4f85200e, entries=150, sequenceid=196, filesize=11.9 K 2024-11-20T10:24:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,684 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for a4b00f80e7a706f41be54575cc7fbb32 in 538ms, sequenceid=196, compaction requested=false 2024-11-20T10:24:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:50,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:50,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-20T10:24:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-20T10:24:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-20T10:24:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9360 sec 2024-11-20T10:24:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.9420 sec 2024-11-20T10:24:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T10:24:50,907 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,907 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-20T10:24:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T10:24:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,910 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,912 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,917 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/e7e07c5d50954bdeadeb982f25fe29d8 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/e7e07c5d50954bdeadeb982f25fe29d8 2024-11-20T10:24:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,923 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/A of a4b00f80e7a706f41be54575cc7fbb32 into e7e07c5d50954bdeadeb982f25fe29d8(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:50,923 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,923 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/A, priority=13, startTime=1732098290047; duration=0sec 2024-11-20T10:24:50,924 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:50,924 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:A 2024-11-20T10:24:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,939 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T10:24:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,952 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/fde646a1667a488e86b64bc6cbb92305 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/fde646a1667a488e86b64bc6cbb92305 2024-11-20T10:24:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208d378e8df7514898ad4d970f90e0078f_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098290936/Put/seqid=0 2024-11-20T10:24:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,966 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/C of a4b00f80e7a706f41be54575cc7fbb32 into fde646a1667a488e86b64bc6cbb92305(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:50,966 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:50,966 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/C, priority=13, startTime=1732098290047; duration=0sec 2024-11-20T10:24:50,966 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:50,966 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:C 2024-11-20T10:24:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742010_1186 (size=22268) 2024-11-20T10:24:50,969 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,973 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208d378e8df7514898ad4d970f90e0078f_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208d378e8df7514898ad4d970f90e0078f_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,975 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/a0116a70e4234a86b4a36879c911d509, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:50,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/a0116a70e4234a86b4a36879c911d509 is 175, key is test_row_0/A:col10/1732098290936/Put/seqid=0 2024-11-20T10:24:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:50,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098350986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098350992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098350990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098350992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098350992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T10:24:51,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742011_1187 (size=65673) 2024-11-20T10:24:51,018 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/a0116a70e4234a86b4a36879c911d509 2024-11-20T10:24:51,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/b8495554826b474b949b78bab5c5deb0 is 50, key is test_row_0/B:col10/1732098290936/Put/seqid=0 2024-11-20T10:24:51,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742012_1188 (size=12151) 2024-11-20T10:24:51,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/b8495554826b474b949b78bab5c5deb0 2024-11-20T10:24:51,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/3500fa2f466a41adb7af8bf77297ea3d is 50, key is test_row_0/C:col10/1732098290936/Put/seqid=0 2024-11-20T10:24:51,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T10:24:51,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:51,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:51,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:51,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:51,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:51,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742013_1189 (size=12151) 2024-11-20T10:24:51,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098351097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098351108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098351108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098351108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098351108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T10:24:51,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:51,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:51,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:51,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:51,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T10:24:51,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098351306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098351327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098351327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098351327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098351328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T10:24:51,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:51,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:51,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:51,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:51,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:51,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/3500fa2f466a41adb7af8bf77297ea3d 2024-11-20T10:24:51,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/a0116a70e4234a86b4a36879c911d509 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/a0116a70e4234a86b4a36879c911d509 2024-11-20T10:24:51,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/a0116a70e4234a86b4a36879c911d509, entries=350, sequenceid=211, filesize=64.1 K 2024-11-20T10:24:51,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/b8495554826b474b949b78bab5c5deb0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b8495554826b474b949b78bab5c5deb0 2024-11-20T10:24:51,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b8495554826b474b949b78bab5c5deb0, entries=150, sequenceid=211, filesize=11.9 K 2024-11-20T10:24:51,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/3500fa2f466a41adb7af8bf77297ea3d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/3500fa2f466a41adb7af8bf77297ea3d 2024-11-20T10:24:51,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/3500fa2f466a41adb7af8bf77297ea3d, entries=150, sequenceid=211, filesize=11.9 K 2024-11-20T10:24:51,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a4b00f80e7a706f41be54575cc7fbb32 in 582ms, sequenceid=211, compaction requested=true 2024-11-20T10:24:51,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:51,523 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:51,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:51,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:51,524 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:51,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:51,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:51,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:51,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:51,524 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128293 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:51,524 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/A is initiating minor compaction (all files) 2024-11-20T10:24:51,525 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/A in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:51,525 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/e7e07c5d50954bdeadeb982f25fe29d8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/3a9f8bb1b3f54f0ea60f032e0a7fb1ea, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/a0116a70e4234a86b4a36879c911d509] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=125.3 K 2024-11-20T10:24:51,525 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:51,525 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/e7e07c5d50954bdeadeb982f25fe29d8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/3a9f8bb1b3f54f0ea60f032e0a7fb1ea, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/a0116a70e4234a86b4a36879c911d509] 2024-11-20T10:24:51,525 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:51,525 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7e07c5d50954bdeadeb982f25fe29d8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732098288565 2024-11-20T10:24:51,525 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/B is initiating minor compaction (all files) 2024-11-20T10:24:51,525 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/B in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:51,525 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/e45f3071b1f845fa899422a20e9ca08e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/d735cef392cd429caa5f21664f4b1fab, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b8495554826b474b949b78bab5c5deb0] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=36.0 K 2024-11-20T10:24:51,526 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a9f8bb1b3f54f0ea60f032e0a7fb1ea, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732098288696 2024-11-20T10:24:51,526 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting e45f3071b1f845fa899422a20e9ca08e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732098288565 2024-11-20T10:24:51,526 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0116a70e4234a86b4a36879c911d509, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732098290895 2024-11-20T10:24:51,526 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d735cef392cd429caa5f21664f4b1fab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732098288696 2024-11-20T10:24:51,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T10:24:51,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:51,527 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T10:24:51,528 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b8495554826b474b949b78bab5c5deb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732098290935 2024-11-20T10:24:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T10:24:51,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203f06a847bcab4f0481100e5cb5239453_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098290986/Put/seqid=0 2024-11-20T10:24:51,544 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:51,544 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/739226063b50449b8fda95d445c415ad is 50, key is test_row_0/B:col10/1732098290936/Put/seqid=0 2024-11-20T10:24:51,549 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:51,559 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120273aabe7b9c14af7aa6040e527fa797c_a4b00f80e7a706f41be54575cc7fbb32 store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:51,562 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120273aabe7b9c14af7aa6040e527fa797c_a4b00f80e7a706f41be54575cc7fbb32, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:51,562 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120273aabe7b9c14af7aa6040e527fa797c_a4b00f80e7a706f41be54575cc7fbb32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:51,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742014_1190 (size=12304) 2024-11-20T10:24:51,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:51,577 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203f06a847bcab4f0481100e5cb5239453_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203f06a847bcab4f0481100e5cb5239453_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:51,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/06de95d3c2ca4634a64e86d13c2763bc, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:51,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/06de95d3c2ca4634a64e86d13c2763bc is 175, key is test_row_0/A:col10/1732098290986/Put/seqid=0 2024-11-20T10:24:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:51,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:51,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742016_1192 (size=4469) 2024-11-20T10:24:51,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742015_1191 (size=12663) 2024-11-20T10:24:51,627 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#A#compaction#161 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:51,628 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/6d390dcad49c4cc488e9b60bbe848aa4 is 175, key is test_row_0/A:col10/1732098290936/Put/seqid=0 2024-11-20T10:24:51,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742017_1193 (size=31105) 2024-11-20T10:24:51,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742018_1194 (size=31617) 2024-11-20T10:24:51,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098351647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098351647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098351648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098351648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098351648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098351756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098351757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098351757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098351757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098351757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098351966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098351967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098351967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098351967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098351967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,035 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/739226063b50449b8fda95d445c415ad as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/739226063b50449b8fda95d445c415ad 2024-11-20T10:24:52,040 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/06de95d3c2ca4634a64e86d13c2763bc 2024-11-20T10:24:52,044 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/B of a4b00f80e7a706f41be54575cc7fbb32 into 739226063b50449b8fda95d445c415ad(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:52,045 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:52,045 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/B, priority=13, startTime=1732098291524; duration=0sec 2024-11-20T10:24:52,045 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:52,045 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:B 2024-11-20T10:24:52,045 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T10:24:52,050 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:52,051 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/C is initiating minor compaction (all files) 2024-11-20T10:24:52,052 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/C in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:52,052 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/fde646a1667a488e86b64bc6cbb92305, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/63abb9b1b9a84e14896c784e4f85200e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/3500fa2f466a41adb7af8bf77297ea3d] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=36.0 K 2024-11-20T10:24:52,059 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/6d390dcad49c4cc488e9b60bbe848aa4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/6d390dcad49c4cc488e9b60bbe848aa4 2024-11-20T10:24:52,060 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting fde646a1667a488e86b64bc6cbb92305, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732098288565 2024-11-20T10:24:52,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/c4b1aaea7faa47059ec1bff35031b640 is 50, key is test_row_0/B:col10/1732098290986/Put/seqid=0 2024-11-20T10:24:52,064 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 63abb9b1b9a84e14896c784e4f85200e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732098288696 2024-11-20T10:24:52,066 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/A of a4b00f80e7a706f41be54575cc7fbb32 into 6d390dcad49c4cc488e9b60bbe848aa4(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:52,066 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:52,066 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/A, priority=13, startTime=1732098291523; duration=0sec 2024-11-20T10:24:52,066 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3500fa2f466a41adb7af8bf77297ea3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732098290935 2024-11-20T10:24:52,066 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:52,066 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:A 2024-11-20T10:24:52,078 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#C#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:52,078 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/dd6bd0a97a2b4a3aa0e5862578bb9c3b is 50, key is test_row_0/C:col10/1732098290936/Put/seqid=0 2024-11-20T10:24:52,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742019_1195 (size=12151) 2024-11-20T10:24:52,097 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/c4b1aaea7faa47059ec1bff35031b640 2024-11-20T10:24:52,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742020_1196 (size=12663) 2024-11-20T10:24:52,128 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/dd6bd0a97a2b4a3aa0e5862578bb9c3b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/dd6bd0a97a2b4a3aa0e5862578bb9c3b 2024-11-20T10:24:52,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/afaffd29e4a2467094eac2e078c3529f is 50, key is test_row_0/C:col10/1732098290986/Put/seqid=0 2024-11-20T10:24:52,138 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/C of a4b00f80e7a706f41be54575cc7fbb32 into dd6bd0a97a2b4a3aa0e5862578bb9c3b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:52,138 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:52,138 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/C, priority=13, startTime=1732098291524; duration=0sec 2024-11-20T10:24:52,138 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:52,138 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:C 2024-11-20T10:24:52,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742021_1197 (size=12151) 2024-11-20T10:24:52,162 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/afaffd29e4a2467094eac2e078c3529f 2024-11-20T10:24:52,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/06de95d3c2ca4634a64e86d13c2763bc as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/06de95d3c2ca4634a64e86d13c2763bc 2024-11-20T10:24:52,177 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/06de95d3c2ca4634a64e86d13c2763bc, entries=150, sequenceid=235, filesize=30.4 K 2024-11-20T10:24:52,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/c4b1aaea7faa47059ec1bff35031b640 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/c4b1aaea7faa47059ec1bff35031b640 2024-11-20T10:24:52,185 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/c4b1aaea7faa47059ec1bff35031b640, entries=150, sequenceid=235, filesize=11.9 K 2024-11-20T10:24:52,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/afaffd29e4a2467094eac2e078c3529f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/afaffd29e4a2467094eac2e078c3529f 2024-11-20T10:24:52,195 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/afaffd29e4a2467094eac2e078c3529f, entries=150, sequenceid=235, filesize=11.9 K 2024-11-20T10:24:52,196 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for a4b00f80e7a706f41be54575cc7fbb32 in 669ms, sequenceid=235, compaction requested=false 2024-11-20T10:24:52,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:52,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:52,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-20T10:24:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-20T10:24:52,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-20T10:24:52,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2860 sec 2024-11-20T10:24:52,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.2980 sec 2024-11-20T10:24:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:52,278 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:52,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-20T10:24:52,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120544130d85abe46178d6783995cfed9df_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098292276/Put/seqid=0 2024-11-20T10:24:52,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098352298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098352303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098352306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098352307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098352309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742022_1198 (size=14794) 2024-11-20T10:24:52,330 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:52,336 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120544130d85abe46178d6783995cfed9df_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120544130d85abe46178d6783995cfed9df_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:52,338 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/fa202d6402c0471e9878e3d362b3c903, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:52,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/fa202d6402c0471e9878e3d362b3c903 is 175, key is test_row_0/A:col10/1732098292276/Put/seqid=0 2024-11-20T10:24:52,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742023_1199 (size=39749) 2024-11-20T10:24:52,355 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/fa202d6402c0471e9878e3d362b3c903 2024-11-20T10:24:52,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/63a1f542763243029682dfb52a42d3ea is 50, key is test_row_0/B:col10/1732098292276/Put/seqid=0 2024-11-20T10:24:52,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742024_1200 (size=12151) 2024-11-20T10:24:52,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098352417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098352417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098352417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098352417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098352417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098352627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098352627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098352627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098352627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098352627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/63a1f542763243029682dfb52a42d3ea 2024-11-20T10:24:52,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/e353687f94714f918e8ae3715f81e6e0 is 50, key is test_row_0/C:col10/1732098292276/Put/seqid=0 2024-11-20T10:24:52,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742025_1201 (size=12151) 2024-11-20T10:24:52,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098352937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098352937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098352937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098352937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098352938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T10:24:53,056 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:53,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:53,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-20T10:24:53,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T10:24:53,058 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:53,059 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:53,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T10:24:53,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/e353687f94714f918e8ae3715f81e6e0 2024-11-20T10:24:53,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/fa202d6402c0471e9878e3d362b3c903 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/fa202d6402c0471e9878e3d362b3c903 2024-11-20T10:24:53,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T10:24:53,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:53,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:53,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:53,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:53,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:53,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/fa202d6402c0471e9878e3d362b3c903, entries=200, sequenceid=253, filesize=38.8 K 2024-11-20T10:24:53,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/63a1f542763243029682dfb52a42d3ea as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/63a1f542763243029682dfb52a42d3ea 2024-11-20T10:24:53,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/63a1f542763243029682dfb52a42d3ea, entries=150, sequenceid=253, filesize=11.9 K 2024-11-20T10:24:53,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/e353687f94714f918e8ae3715f81e6e0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/e353687f94714f918e8ae3715f81e6e0 2024-11-20T10:24:53,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/e353687f94714f918e8ae3715f81e6e0, entries=150, sequenceid=253, filesize=11.9 K 2024-11-20T10:24:53,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for a4b00f80e7a706f41be54575cc7fbb32 in 963ms, sequenceid=253, compaction requested=true 2024-11-20T10:24:53,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:53,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:53,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:53,242 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:53,242 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:53,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:53,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:53,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:53,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:53,243 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:53,243 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:53,243 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/B is initiating minor compaction (all files) 2024-11-20T10:24:53,243 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/A is initiating minor compaction (all files) 2024-11-20T10:24:53,244 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/B in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:53,244 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/A in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:53,244 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/6d390dcad49c4cc488e9b60bbe848aa4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/06de95d3c2ca4634a64e86d13c2763bc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/fa202d6402c0471e9878e3d362b3c903] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=100.1 K 2024-11-20T10:24:53,244 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/739226063b50449b8fda95d445c415ad, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/c4b1aaea7faa47059ec1bff35031b640, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/63a1f542763243029682dfb52a42d3ea] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=36.1 K 2024-11-20T10:24:53,244 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:53,244 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/6d390dcad49c4cc488e9b60bbe848aa4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/06de95d3c2ca4634a64e86d13c2763bc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/fa202d6402c0471e9878e3d362b3c903] 2024-11-20T10:24:53,244 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d390dcad49c4cc488e9b60bbe848aa4, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732098290935 2024-11-20T10:24:53,244 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 739226063b50449b8fda95d445c415ad, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732098290935 2024-11-20T10:24:53,248 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06de95d3c2ca4634a64e86d13c2763bc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732098290984 2024-11-20T10:24:53,248 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c4b1aaea7faa47059ec1bff35031b640, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732098290984 2024-11-20T10:24:53,249 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 63a1f542763243029682dfb52a42d3ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732098291645 2024-11-20T10:24:53,249 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa202d6402c0471e9878e3d362b3c903, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732098291645 2024-11-20T10:24:53,264 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:53,265 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/b1b445771f824cb389689d029c228259 is 50, key is test_row_0/B:col10/1732098292276/Put/seqid=0 2024-11-20T10:24:53,277 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:53,305 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112001180bca836a4f2884152d468db9414b_a4b00f80e7a706f41be54575cc7fbb32 store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:53,307 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112001180bca836a4f2884152d468db9414b_a4b00f80e7a706f41be54575cc7fbb32, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:53,307 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112001180bca836a4f2884152d468db9414b_a4b00f80e7a706f41be54575cc7fbb32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:53,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742026_1202 (size=12765) 2024-11-20T10:24:53,336 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/b1b445771f824cb389689d029c228259 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b1b445771f824cb389689d029c228259 2024-11-20T10:24:53,342 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/B of a4b00f80e7a706f41be54575cc7fbb32 into b1b445771f824cb389689d029c228259(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:53,342 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:53,342 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/B, priority=13, startTime=1732098293242; duration=0sec 2024-11-20T10:24:53,342 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:53,342 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:B 2024-11-20T10:24:53,342 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:53,344 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:53,344 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/C is initiating minor compaction (all files) 2024-11-20T10:24:53,344 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/C in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:53,344 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/dd6bd0a97a2b4a3aa0e5862578bb9c3b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/afaffd29e4a2467094eac2e078c3529f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/e353687f94714f918e8ae3715f81e6e0] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=36.1 K 2024-11-20T10:24:53,345 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting dd6bd0a97a2b4a3aa0e5862578bb9c3b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732098290935 2024-11-20T10:24:53,345 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting afaffd29e4a2467094eac2e078c3529f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732098290984 2024-11-20T10:24:53,346 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting e353687f94714f918e8ae3715f81e6e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732098291645 2024-11-20T10:24:53,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742027_1203 (size=4469) 2024-11-20T10:24:53,348 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#A#compaction#169 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:53,349 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/80ed6736f2c046ee999a340ca8e16f65 is 175, key is test_row_0/A:col10/1732098292276/Put/seqid=0 2024-11-20T10:24:53,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T10:24:53,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:53,365 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-20T10:24:53,368 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:53,369 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/eefcfa7be33d4d11ab408d6f1a0e4ec7 is 50, key is test_row_0/C:col10/1732098292276/Put/seqid=0 2024-11-20T10:24:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T10:24:53,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742028_1204 (size=31719) 2024-11-20T10:24:53,423 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/80ed6736f2c046ee999a340ca8e16f65 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/80ed6736f2c046ee999a340ca8e16f65 2024-11-20T10:24:53,435 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/A of a4b00f80e7a706f41be54575cc7fbb32 into 80ed6736f2c046ee999a340ca8e16f65(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:53,435 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:53,435 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/A, priority=13, startTime=1732098293242; duration=0sec 2024-11-20T10:24:53,435 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:53,435 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:A 2024-11-20T10:24:53,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203a1ae6b99e504b53973f9557cf716579_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098292295/Put/seqid=0 2024-11-20T10:24:53,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:53,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:53,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742029_1205 (size=12765) 2024-11-20T10:24:53,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098353467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098353467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098353467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:53,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098353468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:53,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098353469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742030_1206 (size=12454) 2024-11-20T10:24:53,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098353576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098353587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098353586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098353587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098353587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T10:24:53,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098353787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:53,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098353796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:53,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098353796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:53,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098353796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098353797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:53,861 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/eefcfa7be33d4d11ab408d6f1a0e4ec7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/eefcfa7be33d4d11ab408d6f1a0e4ec7 2024-11-20T10:24:53,868 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/C of a4b00f80e7a706f41be54575cc7fbb32 into eefcfa7be33d4d11ab408d6f1a0e4ec7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:53,868 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:53,868 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/C, priority=13, startTime=1732098293242; duration=0sec 2024-11-20T10:24:53,868 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:53,869 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:C 2024-11-20T10:24:53,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:53,886 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203a1ae6b99e504b53973f9557cf716579_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203a1ae6b99e504b53973f9557cf716579_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:53,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/420457ea34354e719790f39e61bd2bd7, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:53,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/420457ea34354e719790f39e61bd2bd7 is 175, key is test_row_0/A:col10/1732098292295/Put/seqid=0 2024-11-20T10:24:53,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742031_1207 (size=31255) 2024-11-20T10:24:54,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098354097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098354106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098354107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098354107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098354117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T10:24:54,333 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/420457ea34354e719790f39e61bd2bd7 2024-11-20T10:24:54,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/88246cd019da49ac8d8db27406d0c9a7 is 50, key is test_row_0/B:col10/1732098292295/Put/seqid=0 2024-11-20T10:24:54,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742032_1208 (size=12301) 2024-11-20T10:24:54,380 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/88246cd019da49ac8d8db27406d0c9a7 2024-11-20T10:24:54,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/bdecf23db50548b1a5ee442ddadb7013 is 50, key is test_row_0/C:col10/1732098292295/Put/seqid=0 2024-11-20T10:24:54,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742033_1209 (size=12301) 2024-11-20T10:24:54,447 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/bdecf23db50548b1a5ee442ddadb7013 2024-11-20T10:24:54,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/420457ea34354e719790f39e61bd2bd7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/420457ea34354e719790f39e61bd2bd7 2024-11-20T10:24:54,463 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/420457ea34354e719790f39e61bd2bd7, entries=150, sequenceid=275, filesize=30.5 K 2024-11-20T10:24:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/88246cd019da49ac8d8db27406d0c9a7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/88246cd019da49ac8d8db27406d0c9a7 2024-11-20T10:24:54,475 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/88246cd019da49ac8d8db27406d0c9a7, entries=150, sequenceid=275, filesize=12.0 K 2024-11-20T10:24:54,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/bdecf23db50548b1a5ee442ddadb7013 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/bdecf23db50548b1a5ee442ddadb7013 2024-11-20T10:24:54,487 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/bdecf23db50548b1a5ee442ddadb7013, entries=150, sequenceid=275, filesize=12.0 K 2024-11-20T10:24:54,491 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for a4b00f80e7a706f41be54575cc7fbb32 in 1126ms, sequenceid=275, compaction requested=false 2024-11-20T10:24:54,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:54,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:54,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-20T10:24:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-20T10:24:54,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-20T10:24:54,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4350 sec 2024-11-20T10:24:54,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4410 sec 2024-11-20T10:24:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:54,612 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:54,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T10:24:54,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e0f63e24515544ae9d472f58d170dfed_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098294611/Put/seqid=0 2024-11-20T10:24:54,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098354654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098354655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098354664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098354666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098354681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742034_1210 (size=14994) 2024-11-20T10:24:54,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098354777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098354777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098354788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098354788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098354797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098354986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098354986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:54,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:54,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098354997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:54,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098354997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098355007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,102 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:55,106 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e0f63e24515544ae9d472f58d170dfed_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e0f63e24515544ae9d472f58d170dfed_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:55,107 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/97de08e74bae4db58b380d1f74824f8e, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:55,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/97de08e74bae4db58b380d1f74824f8e is 175, key is test_row_0/A:col10/1732098294611/Put/seqid=0 2024-11-20T10:24:55,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742035_1211 (size=39949) 2024-11-20T10:24:55,126 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/97de08e74bae4db58b380d1f74824f8e 2024-11-20T10:24:55,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/6ce266577ae04fa09f2e14af97fb2b25 is 50, key is test_row_0/B:col10/1732098294611/Put/seqid=0 2024-11-20T10:24:55,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742036_1212 (size=12301) 2024-11-20T10:24:55,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/6ce266577ae04fa09f2e14af97fb2b25 2024-11-20T10:24:55,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/cb5681174b134be39b3b92bdc91220b4 is 50, key is test_row_0/C:col10/1732098294611/Put/seqid=0 2024-11-20T10:24:55,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742037_1213 (size=12301) 2024-11-20T10:24:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T10:24:55,207 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:55,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-20T10:24:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T10:24:55,210 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:55,210 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:55,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:55,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098355296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098355297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098355307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098355307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T10:24:55,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098355317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T10:24:55,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:55,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:55,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:55,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:55,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:55,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T10:24:55,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:55,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:55,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:55,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:55,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T10:24:55,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/cb5681174b134be39b3b92bdc91220b4 2024-11-20T10:24:55,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/97de08e74bae4db58b380d1f74824f8e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/97de08e74bae4db58b380d1f74824f8e 2024-11-20T10:24:55,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/97de08e74bae4db58b380d1f74824f8e, entries=200, sequenceid=294, filesize=39.0 K 2024-11-20T10:24:55,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/6ce266577ae04fa09f2e14af97fb2b25 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/6ce266577ae04fa09f2e14af97fb2b25 2024-11-20T10:24:55,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/6ce266577ae04fa09f2e14af97fb2b25, entries=150, sequenceid=294, filesize=12.0 K 2024-11-20T10:24:55,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/cb5681174b134be39b3b92bdc91220b4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/cb5681174b134be39b3b92bdc91220b4 2024-11-20T10:24:55,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/cb5681174b134be39b3b92bdc91220b4, entries=150, sequenceid=294, filesize=12.0 K 2024-11-20T10:24:55,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for a4b00f80e7a706f41be54575cc7fbb32 in 1011ms, sequenceid=294, compaction requested=true 2024-11-20T10:24:55,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:55,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:55,624 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:55,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:55,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:55,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:55,624 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:55,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:55,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:55,625 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:55,625 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/A is initiating minor compaction (all files) 2024-11-20T10:24:55,625 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/A in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:55,625 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/80ed6736f2c046ee999a340ca8e16f65, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/420457ea34354e719790f39e61bd2bd7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/97de08e74bae4db58b380d1f74824f8e] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=100.5 K 2024-11-20T10:24:55,626 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:55,626 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/80ed6736f2c046ee999a340ca8e16f65, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/420457ea34354e719790f39e61bd2bd7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/97de08e74bae4db58b380d1f74824f8e] 2024-11-20T10:24:55,626 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:55,626 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/B is initiating minor compaction (all files) 2024-11-20T10:24:55,626 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/B in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:55,626 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b1b445771f824cb389689d029c228259, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/88246cd019da49ac8d8db27406d0c9a7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/6ce266577ae04fa09f2e14af97fb2b25] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=36.5 K 2024-11-20T10:24:55,627 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b1b445771f824cb389689d029c228259, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732098291645 2024-11-20T10:24:55,627 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80ed6736f2c046ee999a340ca8e16f65, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732098291645 2024-11-20T10:24:55,627 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 88246cd019da49ac8d8db27406d0c9a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732098292295 2024-11-20T10:24:55,627 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 420457ea34354e719790f39e61bd2bd7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732098292295 2024-11-20T10:24:55,628 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ce266577ae04fa09f2e14af97fb2b25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732098293464 2024-11-20T10:24:55,628 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97de08e74bae4db58b380d1f74824f8e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732098293464 2024-11-20T10:24:55,645 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:55,649 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#B#compaction#178 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:55,649 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/30e30742ffd64aca8f896f66c88649d3 is 50, key is test_row_0/B:col10/1732098294611/Put/seqid=0 2024-11-20T10:24:55,658 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120cb9ec84478124aeabf0685c2a9689965_a4b00f80e7a706f41be54575cc7fbb32 store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:55,659 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120cb9ec84478124aeabf0685c2a9689965_a4b00f80e7a706f41be54575cc7fbb32, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:55,660 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cb9ec84478124aeabf0685c2a9689965_a4b00f80e7a706f41be54575cc7fbb32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:55,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T10:24:55,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:55,673 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T10:24:55,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742038_1214 (size=13017) 2024-11-20T10:24:55,697 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/30e30742ffd64aca8f896f66c88649d3 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/30e30742ffd64aca8f896f66c88649d3 2024-11-20T10:24:55,706 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/B of a4b00f80e7a706f41be54575cc7fbb32 into 30e30742ffd64aca8f896f66c88649d3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:55,706 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:55,706 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/B, priority=13, startTime=1732098295624; duration=0sec 2024-11-20T10:24:55,706 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:55,707 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:B 2024-11-20T10:24:55,707 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:55,708 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:55,708 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/C is initiating minor compaction (all files) 2024-11-20T10:24:55,708 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/C in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:55,708 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/eefcfa7be33d4d11ab408d6f1a0e4ec7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/bdecf23db50548b1a5ee442ddadb7013, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/cb5681174b134be39b3b92bdc91220b4] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=36.5 K 2024-11-20T10:24:55,708 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting eefcfa7be33d4d11ab408d6f1a0e4ec7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732098291645 2024-11-20T10:24:55,709 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting bdecf23db50548b1a5ee442ddadb7013, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732098292295 2024-11-20T10:24:55,709 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting cb5681174b134be39b3b92bdc91220b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732098293464 2024-11-20T10:24:55,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742039_1215 (size=4469) 2024-11-20T10:24:55,714 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#A#compaction#177 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:55,715 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/7048d02d7f6d4a47a0fbe95dca066819 is 175, key is test_row_0/A:col10/1732098294611/Put/seqid=0 2024-11-20T10:24:55,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a683047010fc4e1d9f56d2c4be6e166c_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098294664/Put/seqid=0 2024-11-20T10:24:55,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742040_1216 (size=31971) 2024-11-20T10:24:55,738 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/7048d02d7f6d4a47a0fbe95dca066819 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/7048d02d7f6d4a47a0fbe95dca066819 2024-11-20T10:24:55,744 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/A of a4b00f80e7a706f41be54575cc7fbb32 into 7048d02d7f6d4a47a0fbe95dca066819(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:55,744 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:55,744 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/A, priority=13, startTime=1732098295624; duration=0sec 2024-11-20T10:24:55,744 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:55,745 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:A 2024-11-20T10:24:55,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742041_1217 (size=12454) 2024-11-20T10:24:55,760 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:55,761 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/fe3d78cee6304561a579f31c1eb71dc0 is 50, key is test_row_0/C:col10/1732098294611/Put/seqid=0 2024-11-20T10:24:55,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742042_1218 (size=13017) 2024-11-20T10:24:55,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:55,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098355830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T10:24:55,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098355831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098355833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098355835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098355836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098355936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098355947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098355947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098355947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:56,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098356146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:56,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:56,152 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a683047010fc4e1d9f56d2c4be6e166c_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a683047010fc4e1d9f56d2c4be6e166c_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:56,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/36402508b2b54133ad32d2ef26943865, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:56,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/36402508b2b54133ad32d2ef26943865 is 175, key is test_row_0/A:col10/1732098294664/Put/seqid=0 2024-11-20T10:24:56,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742043_1219 (size=31255) 2024-11-20T10:24:56,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098356157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:56,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098356157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098356157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:56,179 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/fe3d78cee6304561a579f31c1eb71dc0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/fe3d78cee6304561a579f31c1eb71dc0 2024-11-20T10:24:56,184 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/C of a4b00f80e7a706f41be54575cc7fbb32 into fe3d78cee6304561a579f31c1eb71dc0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:56,184 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:56,184 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/C, priority=13, startTime=1732098295624; duration=0sec 2024-11-20T10:24:56,184 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:56,184 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:C 2024-11-20T10:24:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T10:24:56,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098356457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:56,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098356467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:56,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098356467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:56,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098356467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:56,535 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T10:24:56,558 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/36402508b2b54133ad32d2ef26943865 2024-11-20T10:24:56,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/b2bea1cf5e694aa9b1af868efe4cd5dd is 50, key is test_row_0/B:col10/1732098294664/Put/seqid=0 2024-11-20T10:24:56,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742044_1220 (size=12301) 2024-11-20T10:24:56,595 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/b2bea1cf5e694aa9b1af868efe4cd5dd 2024-11-20T10:24:56,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/b062ebfec0ec4650936f4402a33767ea is 50, key is test_row_0/C:col10/1732098294664/Put/seqid=0 2024-11-20T10:24:56,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742045_1221 (size=12301) 2024-11-20T10:24:56,621 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/b062ebfec0ec4650936f4402a33767ea 2024-11-20T10:24:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/36402508b2b54133ad32d2ef26943865 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/36402508b2b54133ad32d2ef26943865 2024-11-20T10:24:56,642 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/36402508b2b54133ad32d2ef26943865, entries=150, sequenceid=314, filesize=30.5 K 2024-11-20T10:24:56,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/b2bea1cf5e694aa9b1af868efe4cd5dd as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b2bea1cf5e694aa9b1af868efe4cd5dd 2024-11-20T10:24:56,652 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b2bea1cf5e694aa9b1af868efe4cd5dd, entries=150, sequenceid=314, filesize=12.0 K 2024-11-20T10:24:56,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/b062ebfec0ec4650936f4402a33767ea as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/b062ebfec0ec4650936f4402a33767ea 2024-11-20T10:24:56,675 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/b062ebfec0ec4650936f4402a33767ea, entries=150, sequenceid=314, filesize=12.0 K 2024-11-20T10:24:56,676 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for a4b00f80e7a706f41be54575cc7fbb32 in 1003ms, sequenceid=314, compaction requested=false 2024-11-20T10:24:56,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:56,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:56,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-20T10:24:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-20T10:24:56,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-20T10:24:56,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4680 sec 2024-11-20T10:24:56,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.4720 sec 2024-11-20T10:24:56,849 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:24:56,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T10:24:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:56,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a5d11201ef604b57a38d8785ba480016_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098296848/Put/seqid=0 2024-11-20T10:24:56,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742046_1222 (size=12454) 2024-11-20T10:24:56,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098356938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:56,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098356966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:56,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098356976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:56,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098356976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098356977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:57,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098357046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:57,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:57,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:57,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:57,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098357257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:57,270 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:57,274 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a5d11201ef604b57a38d8785ba480016_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a5d11201ef604b57a38d8785ba480016_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:57,275 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/c5ce95de96db428e97e56741ef67709b, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:57,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/c5ce95de96db428e97e56741ef67709b is 175, key is test_row_0/A:col10/1732098296848/Put/seqid=0 2024-11-20T10:24:57,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742047_1223 (size=31255) 2024-11-20T10:24:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T10:24:57,367 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:24:57,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:24:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-20T10:24:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T10:24:57,369 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:24:57,370 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:24:57,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:24:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T10:24:57,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T10:24:57,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:57,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:57,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:57,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:57,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:57,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098357566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:57,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T10:24:57,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:57,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:57,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:57,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:57,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T10:24:57,699 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/c5ce95de96db428e97e56741ef67709b 2024-11-20T10:24:57,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/f983f975f4f840c092dd96d943e6d2f3 is 50, key is test_row_0/B:col10/1732098296848/Put/seqid=0 2024-11-20T10:24:57,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742048_1224 (size=12301) 2024-11-20T10:24:57,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/f983f975f4f840c092dd96d943e6d2f3 2024-11-20T10:24:57,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/d122b855ce7d410ba69df0083fbcf24f is 50, key is test_row_0/C:col10/1732098296848/Put/seqid=0 2024-11-20T10:24:57,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742049_1225 (size=12301) 2024-11-20T10:24:57,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T10:24:57,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:57,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:57,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:57,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:57,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:57,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T10:24:57,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:57,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:57,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:57,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:57,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:57,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:57,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098357987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:57,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:57,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098357987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:57,989 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_1' in a4b00f80e7a706f41be54575cc7fbb32 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:24:57,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:57,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098357987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:57,990 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in a4b00f80e7a706f41be54575cc7fbb32 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:24:57,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:57,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098357987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:57,991 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_0' in a4b00f80e7a706f41be54575cc7fbb32 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:24:57,991 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_1' in a4b00f80e7a706f41be54575cc7fbb32 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:24:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:57,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T10:24:58,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:58,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098358077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:58,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/d122b855ce7d410ba69df0083fbcf24f 2024-11-20T10:24:58,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T10:24:58,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:58,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:58,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:58,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:58,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:24:58,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/c5ce95de96db428e97e56741ef67709b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/c5ce95de96db428e97e56741ef67709b 2024-11-20T10:24:58,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/c5ce95de96db428e97e56741ef67709b, entries=150, sequenceid=334, filesize=30.5 K 2024-11-20T10:24:58,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/f983f975f4f840c092dd96d943e6d2f3 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f983f975f4f840c092dd96d943e6d2f3 2024-11-20T10:24:58,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f983f975f4f840c092dd96d943e6d2f3, entries=150, sequenceid=334, filesize=12.0 K 2024-11-20T10:24:58,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/d122b855ce7d410ba69df0083fbcf24f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/d122b855ce7d410ba69df0083fbcf24f 2024-11-20T10:24:58,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/d122b855ce7d410ba69df0083fbcf24f, entries=150, sequenceid=334, filesize=12.0 K 2024-11-20T10:24:58,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a4b00f80e7a706f41be54575cc7fbb32 in 1309ms, sequenceid=334, compaction requested=true 2024-11-20T10:24:58,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:58,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:24:58,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:58,159 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:58,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:24:58,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:58,159 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:58,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:24:58,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:58,160 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:58,160 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:58,160 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/B is initiating minor compaction (all files) 2024-11-20T10:24:58,160 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/A is initiating minor compaction (all files) 2024-11-20T10:24:58,160 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/B in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:58,160 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/A in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:58,160 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/7048d02d7f6d4a47a0fbe95dca066819, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/36402508b2b54133ad32d2ef26943865, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/c5ce95de96db428e97e56741ef67709b] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=92.3 K 2024-11-20T10:24:58,160 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/30e30742ffd64aca8f896f66c88649d3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b2bea1cf5e694aa9b1af868efe4cd5dd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f983f975f4f840c092dd96d943e6d2f3] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=36.7 K 2024-11-20T10:24:58,160 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:58,160 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/7048d02d7f6d4a47a0fbe95dca066819, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/36402508b2b54133ad32d2ef26943865, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/c5ce95de96db428e97e56741ef67709b] 2024-11-20T10:24:58,160 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 30e30742ffd64aca8f896f66c88649d3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732098293464 2024-11-20T10:24:58,161 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b2bea1cf5e694aa9b1af868efe4cd5dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732098294650 2024-11-20T10:24:58,162 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f983f975f4f840c092dd96d943e6d2f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732098295831 2024-11-20T10:24:58,162 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7048d02d7f6d4a47a0fbe95dca066819, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732098293464 2024-11-20T10:24:58,162 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36402508b2b54133ad32d2ef26943865, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732098294650 2024-11-20T10:24:58,163 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5ce95de96db428e97e56741ef67709b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732098295831 2024-11-20T10:24:58,191 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#B#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:58,192 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/1ba632cc16724e3f9a9b552fc360060c is 50, key is test_row_0/B:col10/1732098296848/Put/seqid=0 2024-11-20T10:24:58,193 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:58,201 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120c7a04ff38df14855818bb15629c857de_a4b00f80e7a706f41be54575cc7fbb32 store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:58,203 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120c7a04ff38df14855818bb15629c857de_a4b00f80e7a706f41be54575cc7fbb32, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:58,203 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c7a04ff38df14855818bb15629c857de_a4b00f80e7a706f41be54575cc7fbb32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:58,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742050_1226 (size=13119) 2024-11-20T10:24:58,216 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/1ba632cc16724e3f9a9b552fc360060c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/1ba632cc16724e3f9a9b552fc360060c 2024-11-20T10:24:58,222 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/B of a4b00f80e7a706f41be54575cc7fbb32 into 1ba632cc16724e3f9a9b552fc360060c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:58,222 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:58,222 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/B, priority=13, startTime=1732098298159; duration=0sec 2024-11-20T10:24:58,222 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:24:58,222 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:B 2024-11-20T10:24:58,222 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:24:58,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742051_1227 (size=4469) 2024-11-20T10:24:58,224 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:24:58,224 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/C is initiating minor compaction (all files) 2024-11-20T10:24:58,224 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/C in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:58,224 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/fe3d78cee6304561a579f31c1eb71dc0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/b062ebfec0ec4650936f4402a33767ea, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/d122b855ce7d410ba69df0083fbcf24f] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=36.7 K 2024-11-20T10:24:58,225 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting fe3d78cee6304561a579f31c1eb71dc0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732098293464 2024-11-20T10:24:58,225 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#A#compaction#187 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:58,225 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/846f554fdd3b4b72bf3c21dfb42706fd is 175, key is test_row_0/A:col10/1732098296848/Put/seqid=0 2024-11-20T10:24:58,226 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b062ebfec0ec4650936f4402a33767ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732098294650 2024-11-20T10:24:58,226 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d122b855ce7d410ba69df0083fbcf24f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732098295831 2024-11-20T10:24:58,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742052_1228 (size=32073) 2024-11-20T10:24:58,241 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/846f554fdd3b4b72bf3c21dfb42706fd as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/846f554fdd3b4b72bf3c21dfb42706fd 2024-11-20T10:24:58,242 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#C#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:24:58,243 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/182e30924e1f45808dbf2025929e5441 is 50, key is test_row_0/C:col10/1732098296848/Put/seqid=0 2024-11-20T10:24:58,255 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/A of a4b00f80e7a706f41be54575cc7fbb32 into 846f554fdd3b4b72bf3c21dfb42706fd(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:58,255 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:58,255 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/A, priority=13, startTime=1732098298159; duration=0sec 2024-11-20T10:24:58,255 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:58,255 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:A 2024-11-20T10:24:58,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742053_1229 (size=13119) 2024-11-20T10:24:58,274 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/182e30924e1f45808dbf2025929e5441 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/182e30924e1f45808dbf2025929e5441 2024-11-20T10:24:58,282 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/C of a4b00f80e7a706f41be54575cc7fbb32 into 182e30924e1f45808dbf2025929e5441(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:24:58,282 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:24:58,282 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/C, priority=13, startTime=1732098298159; duration=0sec 2024-11-20T10:24:58,282 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:24:58,282 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:C 2024-11-20T10:24:58,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T10:24:58,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:24:58,288 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T10:24:58,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dd78970552984ff3a088a4f627d29c94_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098296904/Put/seqid=0 2024-11-20T10:24:58,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742054_1230 (size=12454) 2024-11-20T10:24:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T10:24:58,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:24:58,723 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dd78970552984ff3a088a4f627d29c94_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dd78970552984ff3a088a4f627d29c94_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:58,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/54bd73bede984f5ab60275bfdeef98dc, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:24:58,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/54bd73bede984f5ab60275bfdeef98dc is 175, key is test_row_0/A:col10/1732098296904/Put/seqid=0 2024-11-20T10:24:58,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742055_1231 (size=31255) 2024-11-20T10:24:59,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:24:59,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:24:59,147 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/54bd73bede984f5ab60275bfdeef98dc 2024-11-20T10:24:59,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098359149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:59,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/abe645e82f0f4b69abc9bec2cc2d1289 is 50, key is test_row_0/B:col10/1732098296904/Put/seqid=0 2024-11-20T10:24:59,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742056_1232 (size=12301) 2024-11-20T10:24:59,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098359257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:59,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098359466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T10:24:59,561 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/abe645e82f0f4b69abc9bec2cc2d1289 2024-11-20T10:24:59,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/a07ea36501944f7f9054245e156345c5 is 50, key is test_row_0/C:col10/1732098296904/Put/seqid=0 2024-11-20T10:24:59,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742057_1233 (size=12301) 2024-11-20T10:24:59,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098359777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:24:59,973 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/a07ea36501944f7f9054245e156345c5 2024-11-20T10:24:59,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/54bd73bede984f5ab60275bfdeef98dc as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/54bd73bede984f5ab60275bfdeef98dc 2024-11-20T10:24:59,984 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/54bd73bede984f5ab60275bfdeef98dc, entries=150, sequenceid=356, filesize=30.5 K 2024-11-20T10:24:59,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/abe645e82f0f4b69abc9bec2cc2d1289 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/abe645e82f0f4b69abc9bec2cc2d1289 2024-11-20T10:24:59,992 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/abe645e82f0f4b69abc9bec2cc2d1289, entries=150, sequenceid=356, filesize=12.0 K 2024-11-20T10:24:59,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/a07ea36501944f7f9054245e156345c5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/a07ea36501944f7f9054245e156345c5 2024-11-20T10:24:59,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:24:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39522 deadline: 1732098359997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:24:59,998 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_1' in a4b00f80e7a706f41be54575cc7fbb32 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:24:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:24:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:00,001 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/a07ea36501944f7f9054245e156345c5, entries=150, sequenceid=356, filesize=12.0 K 2024-11-20T10:25:00,002 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for a4b00f80e7a706f41be54575cc7fbb32 in 1714ms, sequenceid=356, compaction requested=false 2024-11-20T10:25:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:25:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-20T10:25:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-20T10:25:00,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-20T10:25:00,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6340 sec 2024-11-20T10:25:00,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.6390 sec 2024-11-20T10:25:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:00,009 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:00,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T10:25:00,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d0f3c41eac214ac9b7765ca7264e5955_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098300007/Put/seqid=0 2024-11-20T10:25:00,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742058_1234 (size=14994) 2024-11-20T10:25:00,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098360044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:00,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:00,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098360047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:00,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098360047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:00,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098360157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:00,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098360157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:00,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:00,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098360157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:00,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098360286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:00,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098360367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098360367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:00,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098360376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:00,442 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:00,446 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d0f3c41eac214ac9b7765ca7264e5955_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d0f3c41eac214ac9b7765ca7264e5955_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:00,446 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/45b15331b06f4820878af8b05d60448e, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:25:00,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/45b15331b06f4820878af8b05d60448e is 175, key is test_row_0/A:col10/1732098300007/Put/seqid=0 2024-11-20T10:25:00,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742059_1235 (size=39949) 2024-11-20T10:25:00,452 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/45b15331b06f4820878af8b05d60448e 2024-11-20T10:25:00,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/9c237cd041f042ee8717158a04eee45a is 50, key is test_row_0/B:col10/1732098300007/Put/seqid=0 2024-11-20T10:25:00,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742060_1236 (size=12301) 2024-11-20T10:25:00,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098360677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:00,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098360677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:00,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098360686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:00,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:00,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:00,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/9c237cd041f042ee8717158a04eee45a 2024-11-20T10:25:00,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/23de3eb4ef2b404e8a358969d540b3c6 is 50, key is test_row_0/C:col10/1732098300007/Put/seqid=0 2024-11-20T10:25:00,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742061_1237 (size=12301) 2024-11-20T10:25:00,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/23de3eb4ef2b404e8a358969d540b3c6 2024-11-20T10:25:00,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/45b15331b06f4820878af8b05d60448e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/45b15331b06f4820878af8b05d60448e 2024-11-20T10:25:00,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/45b15331b06f4820878af8b05d60448e, entries=200, sequenceid=374, filesize=39.0 K 2024-11-20T10:25:00,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/9c237cd041f042ee8717158a04eee45a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/9c237cd041f042ee8717158a04eee45a 2024-11-20T10:25:00,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/9c237cd041f042ee8717158a04eee45a, entries=150, sequenceid=374, filesize=12.0 K 2024-11-20T10:25:00,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/23de3eb4ef2b404e8a358969d540b3c6 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/23de3eb4ef2b404e8a358969d540b3c6 2024-11-20T10:25:00,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/23de3eb4ef2b404e8a358969d540b3c6, entries=150, sequenceid=374, filesize=12.0 K 2024-11-20T10:25:00,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for a4b00f80e7a706f41be54575cc7fbb32 in 909ms, sequenceid=374, compaction requested=true 2024-11-20T10:25:00,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:25:00,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:25:00,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:00,918 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:00,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:25:00,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:00,918 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:00,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4b00f80e7a706f41be54575cc7fbb32:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:25:00,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:00,921 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:00,921 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/B is initiating minor compaction (all files) 2024-11-20T10:25:00,921 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/B in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:00,922 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/1ba632cc16724e3f9a9b552fc360060c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/abe645e82f0f4b69abc9bec2cc2d1289, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/9c237cd041f042ee8717158a04eee45a] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=36.8 K 2024-11-20T10:25:00,922 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:00,922 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/A is initiating minor compaction (all files) 2024-11-20T10:25:00,922 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/A in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:00,922 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/846f554fdd3b4b72bf3c21dfb42706fd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/54bd73bede984f5ab60275bfdeef98dc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/45b15331b06f4820878af8b05d60448e] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=100.9 K 2024-11-20T10:25:00,922 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:00,922 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/846f554fdd3b4b72bf3c21dfb42706fd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/54bd73bede984f5ab60275bfdeef98dc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/45b15331b06f4820878af8b05d60448e] 2024-11-20T10:25:00,923 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 846f554fdd3b4b72bf3c21dfb42706fd, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732098295831 2024-11-20T10:25:00,923 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ba632cc16724e3f9a9b552fc360060c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732098295831 2024-11-20T10:25:00,924 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54bd73bede984f5ab60275bfdeef98dc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732098296904 2024-11-20T10:25:00,924 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting abe645e82f0f4b69abc9bec2cc2d1289, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732098296904 2024-11-20T10:25:00,925 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c237cd041f042ee8717158a04eee45a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732098299143 2024-11-20T10:25:00,925 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45b15331b06f4820878af8b05d60448e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732098299136 2024-11-20T10:25:00,932 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:25:00,934 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:00,935 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/416c26df9b5e4e5e86592c95fd2d7cbf is 50, key is test_row_0/B:col10/1732098300007/Put/seqid=0 2024-11-20T10:25:00,945 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120735704e3a92444e2a10fa9e7068f8e03_a4b00f80e7a706f41be54575cc7fbb32 store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:25:00,947 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120735704e3a92444e2a10fa9e7068f8e03_a4b00f80e7a706f41be54575cc7fbb32, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:25:00,947 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120735704e3a92444e2a10fa9e7068f8e03_a4b00f80e7a706f41be54575cc7fbb32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:25:00,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742062_1238 (size=13221) 2024-11-20T10:25:00,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742063_1239 (size=4469) 2024-11-20T10:25:00,956 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#A#compaction#195 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:00,957 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/a2c6ae45bd58477a970da3c24fb336bb is 175, key is test_row_0/A:col10/1732098300007/Put/seqid=0 2024-11-20T10:25:00,961 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/416c26df9b5e4e5e86592c95fd2d7cbf as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/416c26df9b5e4e5e86592c95fd2d7cbf 2024-11-20T10:25:00,967 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/B of a4b00f80e7a706f41be54575cc7fbb32 into 416c26df9b5e4e5e86592c95fd2d7cbf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:00,967 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:25:00,968 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/B, priority=13, startTime=1732098300918; duration=0sec 2024-11-20T10:25:00,968 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:00,968 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:B 2024-11-20T10:25:00,968 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:00,969 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:00,969 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a4b00f80e7a706f41be54575cc7fbb32/C is initiating minor compaction (all files) 2024-11-20T10:25:00,969 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4b00f80e7a706f41be54575cc7fbb32/C in TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:00,969 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/182e30924e1f45808dbf2025929e5441, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/a07ea36501944f7f9054245e156345c5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/23de3eb4ef2b404e8a358969d540b3c6] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp, totalSize=36.8 K 2024-11-20T10:25:00,969 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 182e30924e1f45808dbf2025929e5441, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732098295831 2024-11-20T10:25:00,970 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a07ea36501944f7f9054245e156345c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732098296904 2024-11-20T10:25:00,970 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 23de3eb4ef2b404e8a358969d540b3c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732098299143 2024-11-20T10:25:00,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742064_1240 (size=32175) 2024-11-20T10:25:00,980 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4b00f80e7a706f41be54575cc7fbb32#C#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:00,980 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/a2c6ae45bd58477a970da3c24fb336bb as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/a2c6ae45bd58477a970da3c24fb336bb 2024-11-20T10:25:00,981 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/53549b3d96dd4eb9bb52335eb7e19979 is 50, key is test_row_0/C:col10/1732098300007/Put/seqid=0 2024-11-20T10:25:00,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742065_1241 (size=13221) 2024-11-20T10:25:00,990 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/A of a4b00f80e7a706f41be54575cc7fbb32 into a2c6ae45bd58477a970da3c24fb336bb(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:00,990 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:25:00,990 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/A, priority=13, startTime=1732098300918; duration=0sec 2024-11-20T10:25:00,990 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:00,990 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:A 2024-11-20T10:25:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:01,190 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:01,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-20T10:25:01,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b31992e945a74d35b1628da7820e41b5_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098301188/Put/seqid=0 2024-11-20T10:25:01,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098361205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:01,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742066_1242 (size=17534) 2024-11-20T10:25:01,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:01,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098361209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:01,212 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:01,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098361210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:01,218 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b31992e945a74d35b1628da7820e41b5_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b31992e945a74d35b1628da7820e41b5_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:01,219 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/c864c79bab744852ab3a84a536e15129, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:25:01,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/c864c79bab744852ab3a84a536e15129 is 175, key is test_row_0/A:col10/1732098301188/Put/seqid=0 2024-11-20T10:25:01,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742067_1243 (size=48639) 2024-11-20T10:25:01,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39572 deadline: 1732098361307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:01,313 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_1' in a4b00f80e7a706f41be54575cc7fbb32 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:25:01,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:01,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098361318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:01,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098361318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:01,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:01,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:01,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098361318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:01,398 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/53549b3d96dd4eb9bb52335eb7e19979 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/53549b3d96dd4eb9bb52335eb7e19979 2024-11-20T10:25:01,405 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4b00f80e7a706f41be54575cc7fbb32/C of a4b00f80e7a706f41be54575cc7fbb32 into 53549b3d96dd4eb9bb52335eb7e19979(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:01,405 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:25:01,405 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., storeName=a4b00f80e7a706f41be54575cc7fbb32/C, priority=13, startTime=1732098300918; duration=0sec 2024-11-20T10:25:01,405 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:01,405 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4b00f80e7a706f41be54575cc7fbb32:C 2024-11-20T10:25:01,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098361526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:01,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098361526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:01,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098361526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:01,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T10:25:01,543 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:01,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-20T10:25:01,548 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T10:25:01,549 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:01,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:01,629 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=397, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/c864c79bab744852ab3a84a536e15129 2024-11-20T10:25:01,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/ef389591596149ac8d6612ad1ca6de9e is 50, key is test_row_0/B:col10/1732098301188/Put/seqid=0 2024-11-20T10:25:01,649 INFO [Thread-661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-661. 2024-11-20T10:25:01,649 DEBUG [Thread-661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:01,649 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:01,649 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:01,650 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:01,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742068_1244 (size=12301) 2024-11-20T10:25:01,653 INFO [Thread-667 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-667. 2024-11-20T10:25:01,653 DEBUG [Thread-667 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:01,653 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:01,653 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:01,653 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:01,655 INFO [Thread-663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-663. 2024-11-20T10:25:01,655 DEBUG [Thread-663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:01,655 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:01,655 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:01,655 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T10:25:01,657 INFO [Thread-665 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-665. 2024-11-20T10:25:01,658 DEBUG [Thread-665 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:01,658 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:01,658 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:01,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:01,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/ef389591596149ac8d6612ad1ca6de9e 2024-11-20T10:25:01,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/d101eaed249049a385b44b13f07fcc2f is 50, key is test_row_0/C:col10/1732098301188/Put/seqid=0 2024-11-20T10:25:01,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742069_1245 (size=12301) 2024-11-20T10:25:01,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T10:25:01,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:01,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:25:01,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:01,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:01,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:01,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:01,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:01,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39568 deadline: 1732098361837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39544 deadline: 1732098361837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39526 deadline: 1732098361837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:01,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T10:25:01,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:01,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:25:01,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:01,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:01,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T10:25:02,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T10:25:02,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:02,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:25:02,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:02,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:02,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:02,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/d101eaed249049a385b44b13f07fcc2f 2024-11-20T10:25:02,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/c864c79bab744852ab3a84a536e15129 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/c864c79bab744852ab3a84a536e15129 2024-11-20T10:25:02,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/c864c79bab744852ab3a84a536e15129, entries=250, sequenceid=397, filesize=47.5 K 2024-11-20T10:25:02,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/ef389591596149ac8d6612ad1ca6de9e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/ef389591596149ac8d6612ad1ca6de9e 2024-11-20T10:25:02,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/ef389591596149ac8d6612ad1ca6de9e, entries=150, sequenceid=397, filesize=12.0 K 2024-11-20T10:25:02,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/d101eaed249049a385b44b13f07fcc2f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/d101eaed249049a385b44b13f07fcc2f 2024-11-20T10:25:02,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/d101eaed249049a385b44b13f07fcc2f, entries=150, sequenceid=397, filesize=12.0 K 2024-11-20T10:25:02,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for a4b00f80e7a706f41be54575cc7fbb32 in 903ms, sequenceid=397, compaction requested=false 2024-11-20T10:25:02,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:25:02,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T10:25:02,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:02,162 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-20T10:25:02,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208ba8d3c789e34eda89e8b7204d7e5ff2_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_0/A:col10/1732098301199/Put/seqid=0 2024-11-20T10:25:02,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742070_1246 (size=12454) 2024-11-20T10:25:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T10:25:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:02,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. as already flushing 2024-11-20T10:25:02,348 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-20T10:25:02,348 INFO [Thread-654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-654. 2024-11-20T10:25:02,348 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-11-20T10:25:02,348 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:02,348 DEBUG [Thread-654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:02,348 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:02,348 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:02,348 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:02,348 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:02,349 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:02,349 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:02,349 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:02,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:02,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:02,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:02,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:02,577 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208ba8d3c789e34eda89e8b7204d7e5ff2_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208ba8d3c789e34eda89e8b7204d7e5ff2_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:02,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/61834b66214e460cb65a24d2c24a4954, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:25:02,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/61834b66214e460cb65a24d2c24a4954 is 175, key is test_row_0/A:col10/1732098301199/Put/seqid=0 2024-11-20T10:25:02,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742071_1247 (size=31255) 2024-11-20T10:25:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T10:25:02,989 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/61834b66214e460cb65a24d2c24a4954 2024-11-20T10:25:02,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/f952b38009d84afe8f03fe4dbe906a55 is 50, key is test_row_0/B:col10/1732098301199/Put/seqid=0 2024-11-20T10:25:03,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742072_1248 (size=12301) 2024-11-20T10:25:03,342 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-20T10:25:03,342 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:03,342 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:03,342 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:03,343 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:03,403 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/f952b38009d84afe8f03fe4dbe906a55 2024-11-20T10:25:03,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/545365e8b3664501b0f2b60d5f51105e is 50, key is test_row_0/C:col10/1732098301199/Put/seqid=0 2024-11-20T10:25:03,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742073_1249 (size=12301) 2024-11-20T10:25:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T10:25:03,825 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/545365e8b3664501b0f2b60d5f51105e 2024-11-20T10:25:03,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/61834b66214e460cb65a24d2c24a4954 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/61834b66214e460cb65a24d2c24a4954 2024-11-20T10:25:03,840 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/61834b66214e460cb65a24d2c24a4954, entries=150, sequenceid=413, filesize=30.5 K 2024-11-20T10:25:03,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/f952b38009d84afe8f03fe4dbe906a55 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f952b38009d84afe8f03fe4dbe906a55 2024-11-20T10:25:03,845 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f952b38009d84afe8f03fe4dbe906a55, entries=150, sequenceid=413, filesize=12.0 K 2024-11-20T10:25:03,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/545365e8b3664501b0f2b60d5f51105e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/545365e8b3664501b0f2b60d5f51105e 2024-11-20T10:25:03,852 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/545365e8b3664501b0f2b60d5f51105e, entries=150, sequenceid=413, filesize=12.0 K 2024-11-20T10:25:03,853 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=26.84 KB/27480 for a4b00f80e7a706f41be54575cc7fbb32 in 1691ms, sequenceid=413, compaction requested=true 2024-11-20T10:25:03,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for a4b00f80e7a706f41be54575cc7fbb32: 2024-11-20T10:25:03,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:03,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-20T10:25:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-20T10:25:03,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-20T10:25:03,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3060 sec 2024-11-20T10:25:03,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.3130 sec 2024-11-20T10:25:04,018 INFO [Thread-656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-656. 2024-11-20T10:25:04,018 DEBUG [Thread-656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:04,018 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:04,018 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:04,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T10:25:05,717 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:05,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T10:25:05,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-20T10:25:05,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-20T10:25:05,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-20T10:25:05,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-20T10:25:05,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-20T10:25:05,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T10:25:05,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3378 2024-11-20T10:25:05,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3312 2024-11-20T10:25:05,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T10:25:05,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1496 2024-11-20T10:25:05,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4488 rows 2024-11-20T10:25:05,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1494 2024-11-20T10:25:05,717 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4481 rows 2024-11-20T10:25:05,718 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T10:25:05,718 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T10:25:05,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:05,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:05,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:05,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T10:25:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T10:25:05,726 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098305726"}]},"ts":"1732098305726"} 2024-11-20T10:25:05,729 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T10:25:05,729 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T10:25:05,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T10:25:05,732 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4b00f80e7a706f41be54575cc7fbb32, UNASSIGN}] 2024-11-20T10:25:05,733 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4b00f80e7a706f41be54575cc7fbb32, UNASSIGN 2024-11-20T10:25:05,734 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a4b00f80e7a706f41be54575cc7fbb32, regionState=CLOSING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:25:05,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4b00f80e7a706f41be54575cc7fbb32, UNASSIGN because future has completed 2024-11-20T10:25:05,735 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T10:25:05,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:25:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T10:25:05,889 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:05,889 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T10:25:05,889 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing a4b00f80e7a706f41be54575cc7fbb32, disabling compactions & flushes 2024-11-20T10:25:05,889 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:05,889 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:05,889 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. after waiting 0 ms 2024-11-20T10:25:05,889 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:05,890 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing a4b00f80e7a706f41be54575cc7fbb32 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T10:25:05,901 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207a390d639ce54091b5713d13b0d0add3_a4b00f80e7a706f41be54575cc7fbb32 is 50, key is test_row_1/A:col10/1732098304017/Put/seqid=0 2024-11-20T10:25:05,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742074_1250 (size=9914) 2024-11-20T10:25:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T10:25:06,307 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:06,318 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207a390d639ce54091b5713d13b0d0add3_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207a390d639ce54091b5713d13b0d0add3_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:06,320 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/217ba2a9d819429eafe858ac550111e6, store: [table=TestAcidGuarantees family=A region=a4b00f80e7a706f41be54575cc7fbb32] 2024-11-20T10:25:06,320 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/217ba2a9d819429eafe858ac550111e6 is 175, key is test_row_1/A:col10/1732098304017/Put/seqid=0 2024-11-20T10:25:06,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742075_1251 (size=22561) 2024-11-20T10:25:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T10:25:06,725 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/217ba2a9d819429eafe858ac550111e6 2024-11-20T10:25:06,737 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/2d6aad85fce6455abb6ecde90875d0ea is 50, key is test_row_1/B:col10/1732098304017/Put/seqid=0 2024-11-20T10:25:06,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742076_1252 (size=9857) 2024-11-20T10:25:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T10:25:07,144 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/2d6aad85fce6455abb6ecde90875d0ea 2024-11-20T10:25:07,158 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/05c1ba28c1e74467b70f3b5a5b890d14 is 50, key is test_row_1/C:col10/1732098304017/Put/seqid=0 2024-11-20T10:25:07,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742077_1253 (size=9857) 2024-11-20T10:25:07,564 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/05c1ba28c1e74467b70f3b5a5b890d14 2024-11-20T10:25:07,575 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/A/217ba2a9d819429eafe858ac550111e6 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/217ba2a9d819429eafe858ac550111e6 2024-11-20T10:25:07,580 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/217ba2a9d819429eafe858ac550111e6, entries=100, sequenceid=421, filesize=22.0 K 2024-11-20T10:25:07,581 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/B/2d6aad85fce6455abb6ecde90875d0ea as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/2d6aad85fce6455abb6ecde90875d0ea 2024-11-20T10:25:07,585 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/2d6aad85fce6455abb6ecde90875d0ea, entries=100, sequenceid=421, filesize=9.6 K 2024-11-20T10:25:07,586 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/.tmp/C/05c1ba28c1e74467b70f3b5a5b890d14 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/05c1ba28c1e74467b70f3b5a5b890d14 2024-11-20T10:25:07,590 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/05c1ba28c1e74467b70f3b5a5b890d14, entries=100, sequenceid=421, filesize=9.6 K 2024-11-20T10:25:07,591 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a4b00f80e7a706f41be54575cc7fbb32 in 1702ms, sequenceid=421, compaction requested=true 2024-11-20T10:25:07,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/aa80b20109c744609b84e7eb6a49ca4f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/1321e7484d9e4f8caadb491ead79c432, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/4ef62091e21647c589e01249fa790249, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/fec1e71d117941979637b68123bfecd3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/f7f66625a52641f7abcb02c708c895a9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/7482635e304143d7af513e4241abdb05, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/02f4c4fb91c542a2ad2a5eb9a64a3bdf, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/f219e7e106564862b963f73952e6a9d7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/5e6bb6bcc9cf4801bb3b081a80b3d20e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/b6c2fcd8868b4e36baa68e9d2d7827e9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/4b79545a45d948fb9d321e722a631669, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/35764acac8894c1c9ad6d752b24dfd1a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/e7e07c5d50954bdeadeb982f25fe29d8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/3a9f8bb1b3f54f0ea60f032e0a7fb1ea, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/a0116a70e4234a86b4a36879c911d509, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/6d390dcad49c4cc488e9b60bbe848aa4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/06de95d3c2ca4634a64e86d13c2763bc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/fa202d6402c0471e9878e3d362b3c903, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/80ed6736f2c046ee999a340ca8e16f65, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/420457ea34354e719790f39e61bd2bd7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/97de08e74bae4db58b380d1f74824f8e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/7048d02d7f6d4a47a0fbe95dca066819, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/36402508b2b54133ad32d2ef26943865, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/846f554fdd3b4b72bf3c21dfb42706fd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/c5ce95de96db428e97e56741ef67709b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/54bd73bede984f5ab60275bfdeef98dc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/45b15331b06f4820878af8b05d60448e] to archive 2024-11-20T10:25:07,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:25:07,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/aa80b20109c744609b84e7eb6a49ca4f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/aa80b20109c744609b84e7eb6a49ca4f 2024-11-20T10:25:07,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/1321e7484d9e4f8caadb491ead79c432 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/1321e7484d9e4f8caadb491ead79c432 2024-11-20T10:25:07,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/4ef62091e21647c589e01249fa790249 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/4ef62091e21647c589e01249fa790249 2024-11-20T10:25:07,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/fec1e71d117941979637b68123bfecd3 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/fec1e71d117941979637b68123bfecd3 2024-11-20T10:25:07,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/f7f66625a52641f7abcb02c708c895a9 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/f7f66625a52641f7abcb02c708c895a9 2024-11-20T10:25:07,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/7482635e304143d7af513e4241abdb05 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/7482635e304143d7af513e4241abdb05 2024-11-20T10:25:07,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/02f4c4fb91c542a2ad2a5eb9a64a3bdf to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/02f4c4fb91c542a2ad2a5eb9a64a3bdf 2024-11-20T10:25:07,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/f219e7e106564862b963f73952e6a9d7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/f219e7e106564862b963f73952e6a9d7 2024-11-20T10:25:07,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/5e6bb6bcc9cf4801bb3b081a80b3d20e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/5e6bb6bcc9cf4801bb3b081a80b3d20e 2024-11-20T10:25:07,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/b6c2fcd8868b4e36baa68e9d2d7827e9 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/b6c2fcd8868b4e36baa68e9d2d7827e9 2024-11-20T10:25:07,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/4b79545a45d948fb9d321e722a631669 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/4b79545a45d948fb9d321e722a631669 2024-11-20T10:25:07,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/35764acac8894c1c9ad6d752b24dfd1a to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/35764acac8894c1c9ad6d752b24dfd1a 2024-11-20T10:25:07,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/e7e07c5d50954bdeadeb982f25fe29d8 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/e7e07c5d50954bdeadeb982f25fe29d8 2024-11-20T10:25:07,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/3a9f8bb1b3f54f0ea60f032e0a7fb1ea to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/3a9f8bb1b3f54f0ea60f032e0a7fb1ea 2024-11-20T10:25:07,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/a0116a70e4234a86b4a36879c911d509 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/a0116a70e4234a86b4a36879c911d509 2024-11-20T10:25:07,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/6d390dcad49c4cc488e9b60bbe848aa4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/6d390dcad49c4cc488e9b60bbe848aa4 2024-11-20T10:25:07,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/06de95d3c2ca4634a64e86d13c2763bc to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/06de95d3c2ca4634a64e86d13c2763bc 2024-11-20T10:25:07,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/fa202d6402c0471e9878e3d362b3c903 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/fa202d6402c0471e9878e3d362b3c903 2024-11-20T10:25:07,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/80ed6736f2c046ee999a340ca8e16f65 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/80ed6736f2c046ee999a340ca8e16f65 2024-11-20T10:25:07,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/420457ea34354e719790f39e61bd2bd7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/420457ea34354e719790f39e61bd2bd7 2024-11-20T10:25:07,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/97de08e74bae4db58b380d1f74824f8e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/97de08e74bae4db58b380d1f74824f8e 2024-11-20T10:25:07,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/7048d02d7f6d4a47a0fbe95dca066819 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/7048d02d7f6d4a47a0fbe95dca066819 2024-11-20T10:25:07,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/36402508b2b54133ad32d2ef26943865 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/36402508b2b54133ad32d2ef26943865 2024-11-20T10:25:07,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/846f554fdd3b4b72bf3c21dfb42706fd to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/846f554fdd3b4b72bf3c21dfb42706fd 2024-11-20T10:25:07,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/c5ce95de96db428e97e56741ef67709b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/c5ce95de96db428e97e56741ef67709b 2024-11-20T10:25:07,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/54bd73bede984f5ab60275bfdeef98dc to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/54bd73bede984f5ab60275bfdeef98dc 2024-11-20T10:25:07,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/45b15331b06f4820878af8b05d60448e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/45b15331b06f4820878af8b05d60448e 2024-11-20T10:25:07,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f1e0e20571cb4b6f86c3223a23b3cfb4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/bd98e428b9374a3dbc7610e7ecf45741, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/2487551c3c4340b8b7f21d790c71e92b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8aeea50801bb496a927d448bdcdfce7a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/93e742d915374c60839b9154174ddfdb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/523f065e12054fa18659606499316cb0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8bfeb86640754b6b8cccdc18faabb5bc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/c2c927ae70974d829be0bd2679d3a441, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/06ea10a5d579462a941d4388396a5c39, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8adc118a372c4a46bbbf187dc8a58427, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/5cf2df5535474364b4df50f2defdcf8e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/e45f3071b1f845fa899422a20e9ca08e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f16fa72f1fe94592b3cbdbfcb6408a16, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/d735cef392cd429caa5f21664f4b1fab, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/739226063b50449b8fda95d445c415ad, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b8495554826b474b949b78bab5c5deb0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/c4b1aaea7faa47059ec1bff35031b640, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b1b445771f824cb389689d029c228259, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/63a1f542763243029682dfb52a42d3ea, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/88246cd019da49ac8d8db27406d0c9a7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/30e30742ffd64aca8f896f66c88649d3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/6ce266577ae04fa09f2e14af97fb2b25, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b2bea1cf5e694aa9b1af868efe4cd5dd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/1ba632cc16724e3f9a9b552fc360060c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f983f975f4f840c092dd96d943e6d2f3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/abe645e82f0f4b69abc9bec2cc2d1289, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/9c237cd041f042ee8717158a04eee45a] to archive 2024-11-20T10:25:07,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:25:07,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f1e0e20571cb4b6f86c3223a23b3cfb4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f1e0e20571cb4b6f86c3223a23b3cfb4 2024-11-20T10:25:07,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/bd98e428b9374a3dbc7610e7ecf45741 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/bd98e428b9374a3dbc7610e7ecf45741 2024-11-20T10:25:07,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/2487551c3c4340b8b7f21d790c71e92b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/2487551c3c4340b8b7f21d790c71e92b 2024-11-20T10:25:07,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8aeea50801bb496a927d448bdcdfce7a to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8aeea50801bb496a927d448bdcdfce7a 2024-11-20T10:25:07,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/93e742d915374c60839b9154174ddfdb to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/93e742d915374c60839b9154174ddfdb 2024-11-20T10:25:07,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/523f065e12054fa18659606499316cb0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/523f065e12054fa18659606499316cb0 2024-11-20T10:25:07,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8bfeb86640754b6b8cccdc18faabb5bc to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8bfeb86640754b6b8cccdc18faabb5bc 2024-11-20T10:25:07,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/c2c927ae70974d829be0bd2679d3a441 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/c2c927ae70974d829be0bd2679d3a441 2024-11-20T10:25:07,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/06ea10a5d579462a941d4388396a5c39 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/06ea10a5d579462a941d4388396a5c39 2024-11-20T10:25:07,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8adc118a372c4a46bbbf187dc8a58427 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/8adc118a372c4a46bbbf187dc8a58427 2024-11-20T10:25:07,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/5cf2df5535474364b4df50f2defdcf8e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/5cf2df5535474364b4df50f2defdcf8e 2024-11-20T10:25:07,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/e45f3071b1f845fa899422a20e9ca08e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/e45f3071b1f845fa899422a20e9ca08e 2024-11-20T10:25:07,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f16fa72f1fe94592b3cbdbfcb6408a16 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f16fa72f1fe94592b3cbdbfcb6408a16 2024-11-20T10:25:07,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/d735cef392cd429caa5f21664f4b1fab to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/d735cef392cd429caa5f21664f4b1fab 2024-11-20T10:25:07,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/739226063b50449b8fda95d445c415ad to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/739226063b50449b8fda95d445c415ad 2024-11-20T10:25:07,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b8495554826b474b949b78bab5c5deb0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b8495554826b474b949b78bab5c5deb0 2024-11-20T10:25:07,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/c4b1aaea7faa47059ec1bff35031b640 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/c4b1aaea7faa47059ec1bff35031b640 2024-11-20T10:25:07,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b1b445771f824cb389689d029c228259 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b1b445771f824cb389689d029c228259 2024-11-20T10:25:07,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/63a1f542763243029682dfb52a42d3ea to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/63a1f542763243029682dfb52a42d3ea 2024-11-20T10:25:07,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/88246cd019da49ac8d8db27406d0c9a7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/88246cd019da49ac8d8db27406d0c9a7 2024-11-20T10:25:07,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/30e30742ffd64aca8f896f66c88649d3 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/30e30742ffd64aca8f896f66c88649d3 2024-11-20T10:25:07,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/6ce266577ae04fa09f2e14af97fb2b25 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/6ce266577ae04fa09f2e14af97fb2b25 2024-11-20T10:25:07,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b2bea1cf5e694aa9b1af868efe4cd5dd to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/b2bea1cf5e694aa9b1af868efe4cd5dd 2024-11-20T10:25:07,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/1ba632cc16724e3f9a9b552fc360060c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/1ba632cc16724e3f9a9b552fc360060c 2024-11-20T10:25:07,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f983f975f4f840c092dd96d943e6d2f3 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f983f975f4f840c092dd96d943e6d2f3 2024-11-20T10:25:07,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/abe645e82f0f4b69abc9bec2cc2d1289 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/abe645e82f0f4b69abc9bec2cc2d1289 2024-11-20T10:25:07,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/9c237cd041f042ee8717158a04eee45a to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/9c237cd041f042ee8717158a04eee45a 2024-11-20T10:25:07,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/350c4f9f759d49ff880320fe83915e16, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/9d9fedfda1164f8781070cca9eb8eca5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/417e37301aed406297c2d72f120ed849, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/90fa0e76f2ff4f64b27fa17da42c59b6, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/8e2f53eb513c4529a0aee4381ac96585, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/98f847fa9a2a4c7db28eaebd13848d9e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/e3c04151ecb54b62b0b2608d63d6024d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/95cd41b3a181402d887c329fb97aa974, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/3a3799442c8248658ea6243be2d2b8cd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/88e83c66c1d748ecbfaed40cd77dcfb4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/554b3d5c95fe493784cbc67767b889f5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/fde646a1667a488e86b64bc6cbb92305, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/56955b84639e47e5be9638d8283cec38, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/63abb9b1b9a84e14896c784e4f85200e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/dd6bd0a97a2b4a3aa0e5862578bb9c3b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/3500fa2f466a41adb7af8bf77297ea3d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/afaffd29e4a2467094eac2e078c3529f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/eefcfa7be33d4d11ab408d6f1a0e4ec7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/e353687f94714f918e8ae3715f81e6e0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/bdecf23db50548b1a5ee442ddadb7013, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/fe3d78cee6304561a579f31c1eb71dc0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/cb5681174b134be39b3b92bdc91220b4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/b062ebfec0ec4650936f4402a33767ea, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/182e30924e1f45808dbf2025929e5441, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/d122b855ce7d410ba69df0083fbcf24f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/a07ea36501944f7f9054245e156345c5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/23de3eb4ef2b404e8a358969d540b3c6] to archive 2024-11-20T10:25:07,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:25:07,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/350c4f9f759d49ff880320fe83915e16 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/350c4f9f759d49ff880320fe83915e16 2024-11-20T10:25:07,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/9d9fedfda1164f8781070cca9eb8eca5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/9d9fedfda1164f8781070cca9eb8eca5 2024-11-20T10:25:07,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/417e37301aed406297c2d72f120ed849 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/417e37301aed406297c2d72f120ed849 2024-11-20T10:25:07,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/90fa0e76f2ff4f64b27fa17da42c59b6 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/90fa0e76f2ff4f64b27fa17da42c59b6 2024-11-20T10:25:07,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/8e2f53eb513c4529a0aee4381ac96585 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/8e2f53eb513c4529a0aee4381ac96585 2024-11-20T10:25:07,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/98f847fa9a2a4c7db28eaebd13848d9e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/98f847fa9a2a4c7db28eaebd13848d9e 2024-11-20T10:25:07,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/e3c04151ecb54b62b0b2608d63d6024d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/e3c04151ecb54b62b0b2608d63d6024d 2024-11-20T10:25:07,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/95cd41b3a181402d887c329fb97aa974 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/95cd41b3a181402d887c329fb97aa974 2024-11-20T10:25:07,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/3a3799442c8248658ea6243be2d2b8cd to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/3a3799442c8248658ea6243be2d2b8cd 2024-11-20T10:25:07,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/88e83c66c1d748ecbfaed40cd77dcfb4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/88e83c66c1d748ecbfaed40cd77dcfb4 2024-11-20T10:25:07,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/554b3d5c95fe493784cbc67767b889f5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/554b3d5c95fe493784cbc67767b889f5 2024-11-20T10:25:07,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/fde646a1667a488e86b64bc6cbb92305 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/fde646a1667a488e86b64bc6cbb92305 2024-11-20T10:25:07,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/56955b84639e47e5be9638d8283cec38 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/56955b84639e47e5be9638d8283cec38 2024-11-20T10:25:07,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/63abb9b1b9a84e14896c784e4f85200e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/63abb9b1b9a84e14896c784e4f85200e 2024-11-20T10:25:07,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/dd6bd0a97a2b4a3aa0e5862578bb9c3b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/dd6bd0a97a2b4a3aa0e5862578bb9c3b 2024-11-20T10:25:07,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/3500fa2f466a41adb7af8bf77297ea3d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/3500fa2f466a41adb7af8bf77297ea3d 2024-11-20T10:25:07,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/afaffd29e4a2467094eac2e078c3529f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/afaffd29e4a2467094eac2e078c3529f 2024-11-20T10:25:07,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/eefcfa7be33d4d11ab408d6f1a0e4ec7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/eefcfa7be33d4d11ab408d6f1a0e4ec7 2024-11-20T10:25:07,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/e353687f94714f918e8ae3715f81e6e0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/e353687f94714f918e8ae3715f81e6e0 2024-11-20T10:25:07,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/bdecf23db50548b1a5ee442ddadb7013 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/bdecf23db50548b1a5ee442ddadb7013 2024-11-20T10:25:07,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/fe3d78cee6304561a579f31c1eb71dc0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/fe3d78cee6304561a579f31c1eb71dc0 2024-11-20T10:25:07,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/cb5681174b134be39b3b92bdc91220b4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/cb5681174b134be39b3b92bdc91220b4 2024-11-20T10:25:07,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/b062ebfec0ec4650936f4402a33767ea to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/b062ebfec0ec4650936f4402a33767ea 2024-11-20T10:25:07,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/182e30924e1f45808dbf2025929e5441 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/182e30924e1f45808dbf2025929e5441 2024-11-20T10:25:07,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/d122b855ce7d410ba69df0083fbcf24f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/d122b855ce7d410ba69df0083fbcf24f 2024-11-20T10:25:07,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/a07ea36501944f7f9054245e156345c5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/a07ea36501944f7f9054245e156345c5 2024-11-20T10:25:07,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/23de3eb4ef2b404e8a358969d540b3c6 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/23de3eb4ef2b404e8a358969d540b3c6 2024-11-20T10:25:07,683 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/recovered.edits/424.seqid, newMaxSeqId=424, maxSeqId=4 2024-11-20T10:25:07,684 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. 2024-11-20T10:25:07,684 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for a4b00f80e7a706f41be54575cc7fbb32: Waiting for close lock at 1732098305889Running coprocessor pre-close hooks at 1732098305889Disabling compacts and flushes for region at 1732098305889Disabling writes for close at 1732098305889Obtaining lock to block concurrent updates at 1732098305890 (+1 ms)Preparing flush snapshotting stores in a4b00f80e7a706f41be54575cc7fbb32 at 1732098305890Finished memstore snapshotting TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732098305890Flushing stores of TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32. at 1732098305892 (+2 ms)Flushing a4b00f80e7a706f41be54575cc7fbb32/A: creating writer at 1732098305892Flushing mob file a4b00f80e7a706f41be54575cc7fbb32/A: appending metadata at 1732098305901 (+9 ms)Flushing mob file a4b00f80e7a706f41be54575cc7fbb32/A: closing flushed file at 1732098305901Flushing a4b00f80e7a706f41be54575cc7fbb32/A: appending metadata at 1732098306320 (+419 ms)Flushing a4b00f80e7a706f41be54575cc7fbb32/A: closing flushed file at 1732098306320Flushing a4b00f80e7a706f41be54575cc7fbb32/B: creating writer at 1732098306732 (+412 ms)Flushing a4b00f80e7a706f41be54575cc7fbb32/B: appending metadata at 1732098306737 (+5 ms)Flushing a4b00f80e7a706f41be54575cc7fbb32/B: closing flushed file at 1732098306737Flushing a4b00f80e7a706f41be54575cc7fbb32/C: creating writer at 1732098307154 (+417 ms)Flushing a4b00f80e7a706f41be54575cc7fbb32/C: appending metadata at 1732098307157 (+3 ms)Flushing a4b00f80e7a706f41be54575cc7fbb32/C: closing flushed file at 1732098307158 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34a87ec4: reopening flushed file at 1732098307574 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2192c685: reopening flushed file at 1732098307580 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57e1dfed: reopening flushed file at 1732098307585 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a4b00f80e7a706f41be54575cc7fbb32 in 1702ms, sequenceid=421, compaction requested=true at 1732098307591 (+6 ms)Writing region close event to WAL at 1732098307680 (+89 ms)Running coprocessor post-close hooks at 1732098307683 (+3 ms)Closed at 1732098307683 2024-11-20T10:25:07,686 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,686 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a4b00f80e7a706f41be54575cc7fbb32, regionState=CLOSED 2024-11-20T10:25:07,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:25:07,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-20T10:25:07,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure a4b00f80e7a706f41be54575cc7fbb32, server=8c533afd331c,35833,1732098238678 in 1.9530 sec 2024-11-20T10:25:07,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-20T10:25:07,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4b00f80e7a706f41be54575cc7fbb32, UNASSIGN in 1.9570 sec 2024-11-20T10:25:07,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-20T10:25:07,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9610 sec 2024-11-20T10:25:07,693 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098307693"}]},"ts":"1732098307693"} 2024-11-20T10:25:07,694 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T10:25:07,694 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T10:25:07,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9760 sec 2024-11-20T10:25:07,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T10:25:07,867 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:07,867 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T10:25:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:07,870 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:07,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T10:25:07,871 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:07,877 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,882 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/recovered.edits] 2024-11-20T10:25:07,887 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/217ba2a9d819429eafe858ac550111e6 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/217ba2a9d819429eafe858ac550111e6 2024-11-20T10:25:07,887 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T10:25:07,888 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/61834b66214e460cb65a24d2c24a4954 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/61834b66214e460cb65a24d2c24a4954 2024-11-20T10:25:07,890 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/a2c6ae45bd58477a970da3c24fb336bb to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/a2c6ae45bd58477a970da3c24fb336bb 2024-11-20T10:25:07,891 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/c864c79bab744852ab3a84a536e15129 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/A/c864c79bab744852ab3a84a536e15129 2024-11-20T10:25:07,893 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/2d6aad85fce6455abb6ecde90875d0ea to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/2d6aad85fce6455abb6ecde90875d0ea 2024-11-20T10:25:07,894 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/416c26df9b5e4e5e86592c95fd2d7cbf to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/416c26df9b5e4e5e86592c95fd2d7cbf 2024-11-20T10:25:07,895 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/ef389591596149ac8d6612ad1ca6de9e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/ef389591596149ac8d6612ad1ca6de9e 2024-11-20T10:25:07,896 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f952b38009d84afe8f03fe4dbe906a55 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/B/f952b38009d84afe8f03fe4dbe906a55 2024-11-20T10:25:07,898 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/05c1ba28c1e74467b70f3b5a5b890d14 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/05c1ba28c1e74467b70f3b5a5b890d14 2024-11-20T10:25:07,899 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/53549b3d96dd4eb9bb52335eb7e19979 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/53549b3d96dd4eb9bb52335eb7e19979 2024-11-20T10:25:07,900 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/545365e8b3664501b0f2b60d5f51105e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/545365e8b3664501b0f2b60d5f51105e 2024-11-20T10:25:07,901 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/d101eaed249049a385b44b13f07fcc2f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/C/d101eaed249049a385b44b13f07fcc2f 2024-11-20T10:25:07,903 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/recovered.edits/424.seqid to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32/recovered.edits/424.seqid 2024-11-20T10:25:07,904 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,904 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T10:25:07,904 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T10:25:07,905 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T10:25:07,908 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200370858ba4354139af58d2cdbde7a2b9_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200370858ba4354139af58d2cdbde7a2b9_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,909 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203a1ae6b99e504b53973f9557cf716579_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203a1ae6b99e504b53973f9557cf716579_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,910 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203f06a847bcab4f0481100e5cb5239453_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203f06a847bcab4f0481100e5cb5239453_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,911 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120544130d85abe46178d6783995cfed9df_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120544130d85abe46178d6783995cfed9df_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,912 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120684e57bbfbd44a23bd422ff7d6a0bb02_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120684e57bbfbd44a23bd422ff7d6a0bb02_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,913 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120733530a295ed497bbe5e2a0b57023767_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120733530a295ed497bbe5e2a0b57023767_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,913 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207a390d639ce54091b5713d13b0d0add3_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207a390d639ce54091b5713d13b0d0add3_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,914 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208acf8086ba7f4a79a94d838b9427bedc_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208acf8086ba7f4a79a94d838b9427bedc_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,915 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208ba8d3c789e34eda89e8b7204d7e5ff2_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208ba8d3c789e34eda89e8b7204d7e5ff2_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,916 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208d378e8df7514898ad4d970f90e0078f_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208d378e8df7514898ad4d970f90e0078f_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,918 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209593962452ea4dca8721af494a1d3b02_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209593962452ea4dca8721af494a1d3b02_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,918 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a5d11201ef604b57a38d8785ba480016_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a5d11201ef604b57a38d8785ba480016_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,919 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a683047010fc4e1d9f56d2c4be6e166c_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a683047010fc4e1d9f56d2c4be6e166c_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,920 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b31992e945a74d35b1628da7820e41b5_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b31992e945a74d35b1628da7820e41b5_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,922 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba2fbce39f0041459e910f9c3e864e39_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba2fbce39f0041459e910f9c3e864e39_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,923 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba7021a9e1914954b20818b72db1ae77_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba7021a9e1914954b20818b72db1ae77_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,923 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c58ae73122284a1aa9d0e1c8392401a0_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c58ae73122284a1aa9d0e1c8392401a0_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,924 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d0f3c41eac214ac9b7765ca7264e5955_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d0f3c41eac214ac9b7765ca7264e5955_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,925 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dd78970552984ff3a088a4f627d29c94_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dd78970552984ff3a088a4f627d29c94_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,926 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e0f63e24515544ae9d472f58d170dfed_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e0f63e24515544ae9d472f58d170dfed_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,927 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120eaaca37c1c02443bbf68a8bae18803a6_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120eaaca37c1c02443bbf68a8bae18803a6_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,928 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f0917a5087aa436380e927a50d8bd957_a4b00f80e7a706f41be54575cc7fbb32 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f0917a5087aa436380e927a50d8bd957_a4b00f80e7a706f41be54575cc7fbb32 2024-11-20T10:25:07,928 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T10:25:07,930 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:07,932 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T10:25:07,934 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T10:25:07,936 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:07,936 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T10:25:07,936 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732098307936"}]},"ts":"9223372036854775807"} 2024-11-20T10:25:07,938 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T10:25:07,938 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a4b00f80e7a706f41be54575cc7fbb32, NAME => 'TestAcidGuarantees,,1732098278466.a4b00f80e7a706f41be54575cc7fbb32.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T10:25:07,938 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T10:25:07,938 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732098307938"}]},"ts":"9223372036854775807"} 2024-11-20T10:25:07,940 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T10:25:07,941 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:07,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 73 msec 2024-11-20T10:25:07,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T10:25:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T10:25:07,977 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:07,984 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 238) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/cluster_df478fc5-be47-d3ac-f609-45ae39353117/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1610357704_22 at /127.0.0.1:54638 [Waiting for operation #469] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/cluster_df478fc5-be47-d3ac-f609-45ae39353117/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1610357704_22 at /127.0.0.1:46182 [Waiting for operation #524] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-611424962_22 at /127.0.0.1:55172 [Waiting for operation #337] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-611424962_22 at /127.0.0.1:55188 [Waiting for operation #353] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=471 (was 474), ProcessCount=11 (was 11), AvailableMemoryMB=8631 (was 8435) - AvailableMemoryMB LEAK? - 2024-11-20T10:25:07,992 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=471, ProcessCount=11, AvailableMemoryMB=8630 2024-11-20T10:25:07,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T10:25:07,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T10:25:07,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:07,995 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T10:25:07,996 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:07,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-20T10:25:07,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-20T10:25:07,996 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T10:25:08,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742078_1254 (size=959) 2024-11-20T10:25:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-20T10:25:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-20T10:25:08,407 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31 2024-11-20T10:25:08,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742079_1255 (size=53) 2024-11-20T10:25:08,419 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:25:08,419 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4030da3517ebc82c8a7cacbbe3bde6e8, disabling compactions & flushes 2024-11-20T10:25:08,419 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:08,419 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:08,419 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. after waiting 0 ms 2024-11-20T10:25:08,419 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:08,419 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:08,419 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4030da3517ebc82c8a7cacbbe3bde6e8: Waiting for close lock at 1732098308419Disabling compacts and flushes for region at 1732098308419Disabling writes for close at 1732098308419Writing region close event to WAL at 1732098308419Closed at 1732098308419 2024-11-20T10:25:08,420 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T10:25:08,421 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732098308420"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732098308420"}]},"ts":"1732098308420"} 2024-11-20T10:25:08,422 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T10:25:08,423 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T10:25:08,423 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098308423"}]},"ts":"1732098308423"} 2024-11-20T10:25:08,424 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T10:25:08,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4030da3517ebc82c8a7cacbbe3bde6e8, ASSIGN}] 2024-11-20T10:25:08,426 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4030da3517ebc82c8a7cacbbe3bde6e8, ASSIGN 2024-11-20T10:25:08,426 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4030da3517ebc82c8a7cacbbe3bde6e8, ASSIGN; state=OFFLINE, location=8c533afd331c,35833,1732098238678; forceNewPlan=false, retain=false 2024-11-20T10:25:08,577 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=4030da3517ebc82c8a7cacbbe3bde6e8, regionState=OPENING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:25:08,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4030da3517ebc82c8a7cacbbe3bde6e8, ASSIGN because future has completed 2024-11-20T10:25:08,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:25:08,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-20T10:25:08,734 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:08,734 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:25:08,735 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:08,735 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:25:08,735 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:08,735 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:08,736 INFO [StoreOpener-4030da3517ebc82c8a7cacbbe3bde6e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:08,738 INFO [StoreOpener-4030da3517ebc82c8a7cacbbe3bde6e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4030da3517ebc82c8a7cacbbe3bde6e8 columnFamilyName A 2024-11-20T10:25:08,738 DEBUG [StoreOpener-4030da3517ebc82c8a7cacbbe3bde6e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:08,738 INFO [StoreOpener-4030da3517ebc82c8a7cacbbe3bde6e8-1 {}] regionserver.HStore(327): Store=4030da3517ebc82c8a7cacbbe3bde6e8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:25:08,738 INFO [StoreOpener-4030da3517ebc82c8a7cacbbe3bde6e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:08,739 INFO [StoreOpener-4030da3517ebc82c8a7cacbbe3bde6e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4030da3517ebc82c8a7cacbbe3bde6e8 columnFamilyName B 2024-11-20T10:25:08,740 DEBUG [StoreOpener-4030da3517ebc82c8a7cacbbe3bde6e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:08,740 INFO [StoreOpener-4030da3517ebc82c8a7cacbbe3bde6e8-1 {}] regionserver.HStore(327): Store=4030da3517ebc82c8a7cacbbe3bde6e8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:25:08,740 INFO [StoreOpener-4030da3517ebc82c8a7cacbbe3bde6e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:08,741 INFO [StoreOpener-4030da3517ebc82c8a7cacbbe3bde6e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4030da3517ebc82c8a7cacbbe3bde6e8 columnFamilyName C 2024-11-20T10:25:08,741 DEBUG [StoreOpener-4030da3517ebc82c8a7cacbbe3bde6e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:08,742 INFO [StoreOpener-4030da3517ebc82c8a7cacbbe3bde6e8-1 {}] regionserver.HStore(327): Store=4030da3517ebc82c8a7cacbbe3bde6e8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:25:08,742 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:08,742 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:08,743 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:08,744 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:08,744 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:08,744 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T10:25:08,745 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:08,747 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:25:08,747 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 4030da3517ebc82c8a7cacbbe3bde6e8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59743950, jitterRate=-0.10974577069282532}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T10:25:08,747 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:08,747 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 4030da3517ebc82c8a7cacbbe3bde6e8: Running coprocessor pre-open hook at 1732098308735Writing region info on filesystem at 1732098308735Initializing all the Stores at 1732098308736 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098308736Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098308736Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098308736Cleaning up temporary data from old regions at 1732098308744 (+8 ms)Running coprocessor post-open hooks at 1732098308747 (+3 ms)Region opened successfully at 1732098308747 2024-11-20T10:25:08,748 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., pid=65, masterSystemTime=1732098308732 2024-11-20T10:25:08,750 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:08,750 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:08,750 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=4030da3517ebc82c8a7cacbbe3bde6e8, regionState=OPEN, openSeqNum=2, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:25:08,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:25:08,754 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-20T10:25:08,754 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 in 173 msec 2024-11-20T10:25:08,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-20T10:25:08,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4030da3517ebc82c8a7cacbbe3bde6e8, ASSIGN in 329 msec 2024-11-20T10:25:08,756 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T10:25:08,756 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098308756"}]},"ts":"1732098308756"} 2024-11-20T10:25:08,757 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T10:25:08,758 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T10:25:08,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 764 msec 2024-11-20T10:25:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-20T10:25:09,137 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:09,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78234ef1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:09,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:09,140 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cfed649, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:09,142 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fb24a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,143 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:09,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:09,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:09,147 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T10:25:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 8c533afd331c,33867,1732098237476 2024-11-20T10:25:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5a1f6d7d 2024-11-20T10:25:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T10:25:09,149 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T10:25:09,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@261f8875, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:09,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:09,152 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5df7fd18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:09,153 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:09,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37998a8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@443e7dc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,155 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:09,155 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:09,155 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168add8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:09,157 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39000c0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aa73d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,158 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:09,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:09,158 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d868c7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:09,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:09,159 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:09,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2140edd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e1cbab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:09,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:09,161 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160ccd19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:09,162 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:09,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62e6a8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a4263d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:09,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:09,163 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:09,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:09,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:09,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ba19292, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:09,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:09,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:09,164 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:09,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c50d1d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ea361c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,165 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:09,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:09,166 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b1d7e2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:09,167 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79413cde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf3bf24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,168 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:09,168 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:09,168 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aa4e698, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:09,169 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:09,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e3d1fcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e88d13e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,170 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:09,170 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:09,171 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:09,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:09,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:09,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@457d46c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:09,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:09,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:09,172 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2fd58f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62fcffc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:09,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:09,173 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:09,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:09,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d61e8f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:09,174 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7181d425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d83ce24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,175 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:09,175 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:09,176 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b8d564, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:09,177 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@631a57a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:09,178 DEBUG [Thread-1188 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:09,178 DEBUG [Thread-1201 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:09,178 DEBUG [Thread-1193 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:09,178 DEBUG [Thread-1186 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:09,179 DEBUG [Thread-1184 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:09,179 DEBUG [Thread-1199 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:09,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:09,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:09,180 DEBUG [Thread-1190 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:09,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:09,181 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:09,181 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:09,181 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:25:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:25:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:25:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:09,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:25:09,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:25:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:25:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:25:09,186 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-20T10:25:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T10:25:09,188 DEBUG [Thread-1197 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:09,189 DEBUG [Thread-1195 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:09,189 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:09,189 DEBUG [Thread-1182 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:09,190 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:09,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:09,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:09,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:09,193 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:09,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:25:09,193 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:09,193 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:25:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:25:09,195 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:25:09,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/05aa028386d6403dbada804abfdcf6f8 is 50, key is test_row_0/A:col10/1732098309189/Put/seqid=0 2024-11-20T10:25:09,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41372 deadline: 1732098369223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098369224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098369226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098369227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1732098369236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742080_1256 (size=12001) 2024-11-20T10:25:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T10:25:09,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41372 deadline: 1732098369336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098369336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098369336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098369336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T10:25:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:09,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:09,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:09,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:09,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1732098369347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T10:25:09,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:09,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:09,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:09,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:09,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T10:25:09,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098369547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41372 deadline: 1732098369547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098369547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098369547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1732098369556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T10:25:09,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:09,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:09,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:09,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:09,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:09,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/05aa028386d6403dbada804abfdcf6f8 2024-11-20T10:25:09,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/c3f08d21fb8341deaa815854942bf58d is 50, key is test_row_0/B:col10/1732098309189/Put/seqid=0 2024-11-20T10:25:09,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742081_1257 (size=12001) 2024-11-20T10:25:09,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T10:25:09,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:09,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:09,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:09,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:09,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T10:25:09,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098369857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41372 deadline: 1732098369857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098369857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098369857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:09,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1732098369866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:09,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:09,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:09,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T10:25:09,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:09,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:09,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:09,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:09,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:10,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/c3f08d21fb8341deaa815854942bf58d 2024-11-20T10:25:10,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T10:25:10,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:10,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:10,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:10,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:10,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:10,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/b40a429142d34b838ee9bec45e88bd8b is 50, key is test_row_0/C:col10/1732098309189/Put/seqid=0 2024-11-20T10:25:10,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742082_1258 (size=12001) 2024-11-20T10:25:10,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T10:25:10,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:10,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:10,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:10,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:10,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T10:25:10,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:10,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098370366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:10,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098370366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:10,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41372 deadline: 1732098370367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:10,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098370367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:10,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1732098370377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:10,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T10:25:10,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:10,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:10,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:10,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:10,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:10,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/b40a429142d34b838ee9bec45e88bd8b 2024-11-20T10:25:10,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/05aa028386d6403dbada804abfdcf6f8 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/05aa028386d6403dbada804abfdcf6f8 2024-11-20T10:25:10,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/05aa028386d6403dbada804abfdcf6f8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T10:25:10,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/c3f08d21fb8341deaa815854942bf58d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/c3f08d21fb8341deaa815854942bf58d 2024-11-20T10:25:10,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/c3f08d21fb8341deaa815854942bf58d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T10:25:10,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/b40a429142d34b838ee9bec45e88bd8b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b40a429142d34b838ee9bec45e88bd8b 2024-11-20T10:25:10,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b40a429142d34b838ee9bec45e88bd8b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T10:25:10,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 1367ms, sequenceid=13, compaction requested=false 2024-11-20T10:25:10,561 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T10:25:10,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:10,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T10:25:10,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:10,575 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T10:25:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/73e90a0a2bc9491eaa88cca468712c73 is 50, key is test_row_0/A:col10/1732098309226/Put/seqid=0 2024-11-20T10:25:10,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742083_1259 (size=12001) 2024-11-20T10:25:10,606 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/73e90a0a2bc9491eaa88cca468712c73 2024-11-20T10:25:10,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/f770b77a17e7456cb15bf70f026be31c is 50, key is test_row_0/B:col10/1732098309226/Put/seqid=0 2024-11-20T10:25:10,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742084_1260 (size=12001) 2024-11-20T10:25:11,034 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/f770b77a17e7456cb15bf70f026be31c 2024-11-20T10:25:11,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/b9d8f00e64aa4f4d94d2ecc22947dc70 is 50, key is test_row_0/C:col10/1732098309226/Put/seqid=0 2024-11-20T10:25:11,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742085_1261 (size=12001) 2024-11-20T10:25:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T10:25:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:11,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:11,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:11,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41372 deadline: 1732098371386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1732098371386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:11,391 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_1' in 4030da3517ebc82c8a7cacbbe3bde6e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:25:11,391 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in 4030da3517ebc82c8a7cacbbe3bde6e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:25:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:11,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098371390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:11,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098371390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098371391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:11,450 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/b9d8f00e64aa4f4d94d2ecc22947dc70 2024-11-20T10:25:11,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/73e90a0a2bc9491eaa88cca468712c73 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/73e90a0a2bc9491eaa88cca468712c73 2024-11-20T10:25:11,457 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/73e90a0a2bc9491eaa88cca468712c73, entries=150, sequenceid=38, filesize=11.7 K 2024-11-20T10:25:11,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/f770b77a17e7456cb15bf70f026be31c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/f770b77a17e7456cb15bf70f026be31c 2024-11-20T10:25:11,462 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/f770b77a17e7456cb15bf70f026be31c, entries=150, sequenceid=38, filesize=11.7 K 2024-11-20T10:25:11,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/b9d8f00e64aa4f4d94d2ecc22947dc70 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b9d8f00e64aa4f4d94d2ecc22947dc70 2024-11-20T10:25:11,466 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b9d8f00e64aa4f4d94d2ecc22947dc70, entries=150, sequenceid=38, filesize=11.7 K 2024-11-20T10:25:11,467 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 892ms, sequenceid=38, compaction requested=false 2024-11-20T10:25:11,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:11,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:11,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-20T10:25:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-20T10:25:11,470 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-20T10:25:11,470 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2790 sec 2024-11-20T10:25:11,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.2850 sec 2024-11-20T10:25:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:11,498 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:11,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T10:25:11,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/3ed2571f69df4d2982a4690986fa337f is 50, key is test_row_0/A:col10/1732098311384/Put/seqid=0 2024-11-20T10:25:11,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742086_1262 (size=14341) 2024-11-20T10:25:11,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/3ed2571f69df4d2982a4690986fa337f 2024-11-20T10:25:11,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/dba1ed103a074dcb96a55779333ccd04 is 50, key is test_row_0/B:col10/1732098311384/Put/seqid=0 2024-11-20T10:25:11,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742087_1263 (size=12001) 2024-11-20T10:25:11,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098371525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:11,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:11,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098371528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:11,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:11,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098371529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:11,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:11,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098371636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:11,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:11,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098371637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:11,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:11,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098371637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:11,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:11,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098371846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:11,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:11,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098371846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:11,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098371846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:11,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/dba1ed103a074dcb96a55779333ccd04 2024-11-20T10:25:11,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/b6335e4dd2da4132935b781d417783c5 is 50, key is test_row_0/C:col10/1732098311384/Put/seqid=0 2024-11-20T10:25:11,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742088_1264 (size=12001) 2024-11-20T10:25:12,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098372157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:12,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098372157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098372157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:12,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/b6335e4dd2da4132935b781d417783c5 2024-11-20T10:25:12,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/3ed2571f69df4d2982a4690986fa337f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/3ed2571f69df4d2982a4690986fa337f 2024-11-20T10:25:12,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/3ed2571f69df4d2982a4690986fa337f, entries=200, sequenceid=53, filesize=14.0 K 2024-11-20T10:25:12,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/dba1ed103a074dcb96a55779333ccd04 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/dba1ed103a074dcb96a55779333ccd04 2024-11-20T10:25:12,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/dba1ed103a074dcb96a55779333ccd04, entries=150, sequenceid=53, filesize=11.7 K 2024-11-20T10:25:12,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/b6335e4dd2da4132935b781d417783c5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b6335e4dd2da4132935b781d417783c5 2024-11-20T10:25:12,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b6335e4dd2da4132935b781d417783c5, entries=150, sequenceid=53, filesize=11.7 K 2024-11-20T10:25:12,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 852ms, sequenceid=53, compaction requested=true 2024-11-20T10:25:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:25:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:25:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:12,350 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:25:12,350 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:12,351 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:12,351 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/A is initiating minor compaction (all files) 2024-11-20T10:25:12,351 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:12,352 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/B is initiating minor compaction (all files) 2024-11-20T10:25:12,352 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/A in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:12,352 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/B in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:12,352 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/05aa028386d6403dbada804abfdcf6f8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/73e90a0a2bc9491eaa88cca468712c73, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/3ed2571f69df4d2982a4690986fa337f] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=37.4 K 2024-11-20T10:25:12,352 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/c3f08d21fb8341deaa815854942bf58d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/f770b77a17e7456cb15bf70f026be31c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/dba1ed103a074dcb96a55779333ccd04] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=35.2 K 2024-11-20T10:25:12,352 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c3f08d21fb8341deaa815854942bf58d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732098309189 2024-11-20T10:25:12,352 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05aa028386d6403dbada804abfdcf6f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732098309189 2024-11-20T10:25:12,352 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73e90a0a2bc9491eaa88cca468712c73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732098309215 2024-11-20T10:25:12,352 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f770b77a17e7456cb15bf70f026be31c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732098309215 2024-11-20T10:25:12,353 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting dba1ed103a074dcb96a55779333ccd04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732098311384 2024-11-20T10:25:12,353 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ed2571f69df4d2982a4690986fa337f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732098311384 2024-11-20T10:25:12,360 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#B#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:12,361 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/3e6c0d9fb80944a7801e648d64b75374 is 50, key is test_row_0/B:col10/1732098311384/Put/seqid=0 2024-11-20T10:25:12,366 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#A#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:12,367 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/c91d2a6724284e0abeb748636482ea4e is 50, key is test_row_0/A:col10/1732098311384/Put/seqid=0 2024-11-20T10:25:12,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742089_1265 (size=12104) 2024-11-20T10:25:12,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742090_1266 (size=12104) 2024-11-20T10:25:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:12,669 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:12,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-20T10:25:12,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/7efb3639427a4a928580a2bdab4b85d0 is 50, key is test_row_0/A:col10/1732098311528/Put/seqid=0 2024-11-20T10:25:12,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742091_1267 (size=14341) 2024-11-20T10:25:12,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:12,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098372684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098372683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:12,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098372686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:12,775 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/3e6c0d9fb80944a7801e648d64b75374 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/3e6c0d9fb80944a7801e648d64b75374 2024-11-20T10:25:12,779 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/c91d2a6724284e0abeb748636482ea4e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/c91d2a6724284e0abeb748636482ea4e 2024-11-20T10:25:12,779 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/B of 4030da3517ebc82c8a7cacbbe3bde6e8 into 3e6c0d9fb80944a7801e648d64b75374(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:12,780 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:12,780 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/B, priority=13, startTime=1732098312350; duration=0sec 2024-11-20T10:25:12,780 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:12,780 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:B 2024-11-20T10:25:12,780 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:12,781 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:12,781 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/C is initiating minor compaction (all files) 2024-11-20T10:25:12,781 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/C in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:12,781 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b40a429142d34b838ee9bec45e88bd8b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b9d8f00e64aa4f4d94d2ecc22947dc70, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b6335e4dd2da4132935b781d417783c5] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=35.2 K 2024-11-20T10:25:12,782 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b40a429142d34b838ee9bec45e88bd8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732098309189 2024-11-20T10:25:12,782 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b9d8f00e64aa4f4d94d2ecc22947dc70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732098309215 2024-11-20T10:25:12,782 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b6335e4dd2da4132935b781d417783c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732098311384 2024-11-20T10:25:12,785 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/A of 4030da3517ebc82c8a7cacbbe3bde6e8 into c91d2a6724284e0abeb748636482ea4e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:12,785 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:12,785 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/A, priority=13, startTime=1732098312350; duration=0sec 2024-11-20T10:25:12,785 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:12,785 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:A 2024-11-20T10:25:12,789 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#C#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:12,790 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/d892aaabec1340b69eef9744785665c7 is 50, key is test_row_0/C:col10/1732098311384/Put/seqid=0 2024-11-20T10:25:12,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742092_1268 (size=12104) 2024-11-20T10:25:12,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098372796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098372797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:12,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098372797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:12,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:12,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:13,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098373006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:13,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098373006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:13,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098373006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:13,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/7efb3639427a4a928580a2bdab4b85d0 2024-11-20T10:25:13,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/8486705598724862b2949489ee915d75 is 50, key is test_row_0/B:col10/1732098311528/Put/seqid=0 2024-11-20T10:25:13,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742093_1269 (size=12001) 2024-11-20T10:25:13,198 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/d892aaabec1340b69eef9744785665c7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/d892aaabec1340b69eef9744785665c7 2024-11-20T10:25:13,203 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/C of 4030da3517ebc82c8a7cacbbe3bde6e8 into d892aaabec1340b69eef9744785665c7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:13,203 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:13,203 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/C, priority=13, startTime=1732098312350; duration=0sec 2024-11-20T10:25:13,203 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:13,204 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:C 2024-11-20T10:25:13,210 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T10:25:13,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098373317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:13,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:13,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098373317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:13,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:13,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:13,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098373317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T10:25:13,346 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:13,347 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-20T10:25:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T10:25:13,349 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:13,350 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:13,350 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:13,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41372 deadline: 1732098373407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:13,409 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in 4030da3517ebc82c8a7cacbbe3bde6e8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:25:13,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:13,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:13,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1732098373417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:13,419 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_1' in 4030da3517ebc82c8a7cacbbe3bde6e8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:25:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T10:25:13,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/8486705598724862b2949489ee915d75 2024-11-20T10:25:13,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/a1adc429c1f5481c9e1d770a69dc5884 is 50, key is test_row_0/C:col10/1732098311528/Put/seqid=0 2024-11-20T10:25:13,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T10:25:13,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:13,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:13,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:13,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:13,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:13,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742094_1270 (size=12001) 2024-11-20T10:25:13,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T10:25:13,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:13,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:13,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:13,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:13,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T10:25:13,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T10:25:13,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:13,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:13,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:13,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:13,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:13,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:13,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:13,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098373827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:13,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098373827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:13,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098373827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:13,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/a1adc429c1f5481c9e1d770a69dc5884 2024-11-20T10:25:13,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/7efb3639427a4a928580a2bdab4b85d0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/7efb3639427a4a928580a2bdab4b85d0 2024-11-20T10:25:13,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/7efb3639427a4a928580a2bdab4b85d0, entries=200, sequenceid=76, filesize=14.0 K 2024-11-20T10:25:13,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/8486705598724862b2949489ee915d75 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8486705598724862b2949489ee915d75 2024-11-20T10:25:13,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8486705598724862b2949489ee915d75, entries=150, sequenceid=76, filesize=11.7 K 2024-11-20T10:25:13,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/a1adc429c1f5481c9e1d770a69dc5884 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a1adc429c1f5481c9e1d770a69dc5884 2024-11-20T10:25:13,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a1adc429c1f5481c9e1d770a69dc5884, entries=150, sequenceid=76, filesize=11.7 K 2024-11-20T10:25:13,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 1271ms, sequenceid=76, compaction requested=false 2024-11-20T10:25:13,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:13,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T10:25:13,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:13,962 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T10:25:13,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/dd9fe2486a9c4389842d299d00ad5fec is 50, key is test_row_0/A:col10/1732098312683/Put/seqid=0 2024-11-20T10:25:13,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742095_1271 (size=12001) 2024-11-20T10:25:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T10:25:14,370 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/dd9fe2486a9c4389842d299d00ad5fec 2024-11-20T10:25:14,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/2ea0a17ff36843a8aa995f952fb513f4 is 50, key is test_row_0/B:col10/1732098312683/Put/seqid=0 2024-11-20T10:25:14,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742096_1272 (size=12001) 2024-11-20T10:25:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T10:25:14,786 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/2ea0a17ff36843a8aa995f952fb513f4 2024-11-20T10:25:14,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/dbe83c21045e4a26807bd6bf7514bc63 is 50, key is test_row_0/C:col10/1732098312683/Put/seqid=0 2024-11-20T10:25:14,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742097_1273 (size=12001) 2024-11-20T10:25:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:14,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:14,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098374878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:14,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:14,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:14,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:14,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098374880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:14,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:14,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:14,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:14,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098374881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:14,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:14,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:14,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:14,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098374986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:14,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098374986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:14,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098374986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:15,198 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/dbe83c21045e4a26807bd6bf7514bc63 2024-11-20T10:25:15,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098375196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098375196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098375196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:15,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/dd9fe2486a9c4389842d299d00ad5fec as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/dd9fe2486a9c4389842d299d00ad5fec 2024-11-20T10:25:15,207 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/dd9fe2486a9c4389842d299d00ad5fec, entries=150, sequenceid=92, filesize=11.7 K 2024-11-20T10:25:15,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/2ea0a17ff36843a8aa995f952fb513f4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/2ea0a17ff36843a8aa995f952fb513f4 2024-11-20T10:25:15,212 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/2ea0a17ff36843a8aa995f952fb513f4, entries=150, sequenceid=92, filesize=11.7 K 2024-11-20T10:25:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/dbe83c21045e4a26807bd6bf7514bc63 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/dbe83c21045e4a26807bd6bf7514bc63 2024-11-20T10:25:15,216 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/dbe83c21045e4a26807bd6bf7514bc63, entries=150, sequenceid=92, filesize=11.7 K 2024-11-20T10:25:15,217 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 1254ms, sequenceid=92, compaction requested=true 2024-11-20T10:25:15,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:15,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:15,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-20T10:25:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-20T10:25:15,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-20T10:25:15,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8680 sec 2024-11-20T10:25:15,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.8730 sec 2024-11-20T10:25:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T10:25:15,497 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:15,497 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-20T10:25:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T10:25:15,499 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:15,499 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:15,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:15,509 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:15,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T10:25:15,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/d537d2c3b1b44243a03700ce8ef535a0 is 50, key is test_row_0/A:col10/1732098315509/Put/seqid=0 2024-11-20T10:25:15,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098375519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:15,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742098_1274 (size=14341) 2024-11-20T10:25:15,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098375525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098375526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T10:25:15,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098375636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:15,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098375636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098375636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:15,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T10:25:15,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:15,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:15,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:15,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:15,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:15,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T10:25:15,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:15,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:15,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:15,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:15,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T10:25:15,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098375846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:15,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098375846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:15,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098375846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:15,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/d537d2c3b1b44243a03700ce8ef535a0 2024-11-20T10:25:15,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/302e464254e74459886892adc423af94 is 50, key is test_row_0/B:col10/1732098315509/Put/seqid=0 2024-11-20T10:25:15,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742099_1275 (size=12001) 2024-11-20T10:25:15,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T10:25:15,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:15,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:15,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:15,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:15,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:16,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T10:25:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:16,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T10:25:16,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098376157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:16,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098376157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:16,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098376157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:16,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T10:25:16,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:16,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:16,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:16,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:16,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:16,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/302e464254e74459886892adc423af94 2024-11-20T10:25:16,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/4985584a4896434eb0a6799e49488cea is 50, key is test_row_0/C:col10/1732098315509/Put/seqid=0 2024-11-20T10:25:16,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742100_1276 (size=12001) 2024-11-20T10:25:16,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/4985584a4896434eb0a6799e49488cea 2024-11-20T10:25:16,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/d537d2c3b1b44243a03700ce8ef535a0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/d537d2c3b1b44243a03700ce8ef535a0 2024-11-20T10:25:16,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/d537d2c3b1b44243a03700ce8ef535a0, entries=200, sequenceid=118, filesize=14.0 K 2024-11-20T10:25:16,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/302e464254e74459886892adc423af94 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/302e464254e74459886892adc423af94 2024-11-20T10:25:16,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/302e464254e74459886892adc423af94, entries=150, sequenceid=118, filesize=11.7 K 2024-11-20T10:25:16,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/4985584a4896434eb0a6799e49488cea as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/4985584a4896434eb0a6799e49488cea 2024-11-20T10:25:16,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/4985584a4896434eb0a6799e49488cea, entries=150, sequenceid=118, filesize=11.7 K 2024-11-20T10:25:16,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 885ms, sequenceid=118, compaction requested=true 2024-11-20T10:25:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:25:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:16,394 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:25:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:25:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:16,395 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:25:16,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:25:16,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:16,396 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:25:16,396 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:25:16,396 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/A is initiating minor compaction (all files) 2024-11-20T10:25:16,396 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/B is initiating minor compaction (all files) 2024-11-20T10:25:16,396 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/B in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:16,396 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/3e6c0d9fb80944a7801e648d64b75374, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8486705598724862b2949489ee915d75, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/2ea0a17ff36843a8aa995f952fb513f4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/302e464254e74459886892adc423af94] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=47.0 K 2024-11-20T10:25:16,396 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/A in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:16,396 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/c91d2a6724284e0abeb748636482ea4e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/7efb3639427a4a928580a2bdab4b85d0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/dd9fe2486a9c4389842d299d00ad5fec, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/d537d2c3b1b44243a03700ce8ef535a0] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=51.5 K 2024-11-20T10:25:16,396 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e6c0d9fb80944a7801e648d64b75374, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732098311384 2024-11-20T10:25:16,397 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c91d2a6724284e0abeb748636482ea4e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732098311384 2024-11-20T10:25:16,397 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8486705598724862b2949489ee915d75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732098311528 2024-11-20T10:25:16,397 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7efb3639427a4a928580a2bdab4b85d0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732098311524 2024-11-20T10:25:16,397 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ea0a17ff36843a8aa995f952fb513f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732098312679 2024-11-20T10:25:16,397 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd9fe2486a9c4389842d299d00ad5fec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732098312679 2024-11-20T10:25:16,398 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 302e464254e74459886892adc423af94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732098314877 2024-11-20T10:25:16,398 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting d537d2c3b1b44243a03700ce8ef535a0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732098314877 2024-11-20T10:25:16,411 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#A#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:16,411 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#B#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:16,411 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/eedbfa1b4d27400da6b25a605e103fee is 50, key is test_row_0/B:col10/1732098315509/Put/seqid=0 2024-11-20T10:25:16,412 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/60890690a10c4bcdaae82bbcc03f9815 is 50, key is test_row_0/A:col10/1732098315509/Put/seqid=0 2024-11-20T10:25:16,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T10:25:16,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:16,415 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:25:16,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742101_1277 (size=12241) 2024-11-20T10:25:16,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/dc691a7ee16f45a994af84be47f9b2d4 is 50, key is test_row_0/A:col10/1732098315524/Put/seqid=0 2024-11-20T10:25:16,438 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/60890690a10c4bcdaae82bbcc03f9815 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/60890690a10c4bcdaae82bbcc03f9815 2024-11-20T10:25:16,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742102_1278 (size=12241) 2024-11-20T10:25:16,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742103_1279 (size=12001) 2024-11-20T10:25:16,448 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/A of 4030da3517ebc82c8a7cacbbe3bde6e8 into 60890690a10c4bcdaae82bbcc03f9815(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:16,448 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:16,448 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/A, priority=12, startTime=1732098316394; duration=0sec 2024-11-20T10:25:16,448 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:16,448 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:A 2024-11-20T10:25:16,448 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:25:16,449 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/dc691a7ee16f45a994af84be47f9b2d4 2024-11-20T10:25:16,451 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:25:16,451 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/C is initiating minor compaction (all files) 2024-11-20T10:25:16,451 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/C in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:16,451 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/d892aaabec1340b69eef9744785665c7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a1adc429c1f5481c9e1d770a69dc5884, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/dbe83c21045e4a26807bd6bf7514bc63, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/4985584a4896434eb0a6799e49488cea] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=47.0 K 2024-11-20T10:25:16,452 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting d892aaabec1340b69eef9744785665c7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732098311384 2024-11-20T10:25:16,452 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1adc429c1f5481c9e1d770a69dc5884, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732098311528 2024-11-20T10:25:16,453 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbe83c21045e4a26807bd6bf7514bc63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732098312679 2024-11-20T10:25:16,454 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4985584a4896434eb0a6799e49488cea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732098314877 2024-11-20T10:25:16,456 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/eedbfa1b4d27400da6b25a605e103fee as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/eedbfa1b4d27400da6b25a605e103fee 2024-11-20T10:25:16,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/9219a71272c34146bb9f29fd86fa9edf is 50, key is test_row_0/B:col10/1732098315524/Put/seqid=0 2024-11-20T10:25:16,464 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/B of 4030da3517ebc82c8a7cacbbe3bde6e8 into eedbfa1b4d27400da6b25a605e103fee(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:16,464 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:16,464 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/B, priority=12, startTime=1732098316394; duration=0sec 2024-11-20T10:25:16,464 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:16,464 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:B 2024-11-20T10:25:16,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742104_1280 (size=12001) 2024-11-20T10:25:16,466 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/9219a71272c34146bb9f29fd86fa9edf 2024-11-20T10:25:16,468 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#C#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:16,468 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/a05e2de6f46a40888bb1ebff57ca4f76 is 50, key is test_row_0/C:col10/1732098315509/Put/seqid=0 2024-11-20T10:25:16,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/2b7ef071bae84f25b342e781b1ff9924 is 50, key is test_row_0/C:col10/1732098315524/Put/seqid=0 2024-11-20T10:25:16,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742105_1281 (size=12241) 2024-11-20T10:25:16,481 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/a05e2de6f46a40888bb1ebff57ca4f76 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a05e2de6f46a40888bb1ebff57ca4f76 2024-11-20T10:25:16,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742106_1282 (size=12001) 2024-11-20T10:25:16,488 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/C of 4030da3517ebc82c8a7cacbbe3bde6e8 into a05e2de6f46a40888bb1ebff57ca4f76(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:16,488 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:16,488 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/C, priority=12, startTime=1732098316395; duration=0sec 2024-11-20T10:25:16,488 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:16,488 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:C 2024-11-20T10:25:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T10:25:16,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:16,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:16,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:16,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098376703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:16,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:16,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098376708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:16,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:16,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098376709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:16,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098376816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:16,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098376816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098376816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:16,883 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/2b7ef071bae84f25b342e781b1ff9924 2024-11-20T10:25:16,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/dc691a7ee16f45a994af84be47f9b2d4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/dc691a7ee16f45a994af84be47f9b2d4 2024-11-20T10:25:16,892 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/dc691a7ee16f45a994af84be47f9b2d4, entries=150, sequenceid=129, filesize=11.7 K 2024-11-20T10:25:16,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/9219a71272c34146bb9f29fd86fa9edf as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/9219a71272c34146bb9f29fd86fa9edf 2024-11-20T10:25:16,897 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/9219a71272c34146bb9f29fd86fa9edf, entries=150, sequenceid=129, filesize=11.7 K 2024-11-20T10:25:16,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/2b7ef071bae84f25b342e781b1ff9924 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2b7ef071bae84f25b342e781b1ff9924 2024-11-20T10:25:16,902 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2b7ef071bae84f25b342e781b1ff9924, entries=150, sequenceid=129, filesize=11.7 K 2024-11-20T10:25:16,903 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 488ms, sequenceid=129, compaction requested=false 2024-11-20T10:25:16,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:16,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:16,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-20T10:25:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-20T10:25:16,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-20T10:25:16,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4060 sec 2024-11-20T10:25:16,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4100 sec 2024-11-20T10:25:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:17,029 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:17,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-20T10:25:17,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/9e93adcc2e0f4cdd8169ac57bfb10ba8 is 50, key is test_row_0/A:col10/1732098317029/Put/seqid=0 2024-11-20T10:25:17,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098377035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:17,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742107_1283 (size=14541) 2024-11-20T10:25:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:17,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098377039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:17,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098377039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:17,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:17,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:17,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:17,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098377146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098377146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:17,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098377146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:17,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:17,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098377357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098377357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098377357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:17,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:17,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/9e93adcc2e0f4cdd8169ac57bfb10ba8 2024-11-20T10:25:17,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1732098377437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:17,440 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_1' in 4030da3517ebc82c8a7cacbbe3bde6e8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:25:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:17,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:17,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41372 deadline: 1732098377446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:17,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/bd9878d2f0444396b416003de988d47d is 50, key is test_row_0/B:col10/1732098317029/Put/seqid=0 2024-11-20T10:25:17,448 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in 4030da3517ebc82c8a7cacbbe3bde6e8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:25:17,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:17,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742108_1284 (size=12151) 2024-11-20T10:25:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T10:25:17,647 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:17,647 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:17,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-20T10:25:17,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T10:25:17,649 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:17,650 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:17,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:17,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098377667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:17,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098377667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:17,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098377667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T10:25:17,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T10:25:17,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:17,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:17,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:17,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:17,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:17,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/bd9878d2f0444396b416003de988d47d 2024-11-20T10:25:17,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/a3afd8bd973b458cbe6576f3a08ef27a is 50, key is test_row_0/C:col10/1732098317029/Put/seqid=0 2024-11-20T10:25:17,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742109_1285 (size=12151) 2024-11-20T10:25:17,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/a3afd8bd973b458cbe6576f3a08ef27a 2024-11-20T10:25:17,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/9e93adcc2e0f4cdd8169ac57bfb10ba8 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/9e93adcc2e0f4cdd8169ac57bfb10ba8 2024-11-20T10:25:17,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/9e93adcc2e0f4cdd8169ac57bfb10ba8, entries=200, sequenceid=161, filesize=14.2 K 2024-11-20T10:25:17,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/bd9878d2f0444396b416003de988d47d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/bd9878d2f0444396b416003de988d47d 2024-11-20T10:25:17,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/bd9878d2f0444396b416003de988d47d, entries=150, sequenceid=161, filesize=11.9 K 2024-11-20T10:25:17,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/a3afd8bd973b458cbe6576f3a08ef27a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a3afd8bd973b458cbe6576f3a08ef27a 2024-11-20T10:25:17,887 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T10:25:17,887 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T10:25:17,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a3afd8bd973b458cbe6576f3a08ef27a, entries=150, sequenceid=161, filesize=11.9 K 2024-11-20T10:25:17,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 860ms, sequenceid=161, compaction requested=true 2024-11-20T10:25:17,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:17,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:25:17,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:17,890 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:17,890 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:17,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:25:17,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:17,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:25:17,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:17,891 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:17,892 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/B is initiating minor compaction (all files) 2024-11-20T10:25:17,892 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/B in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:17,892 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/eedbfa1b4d27400da6b25a605e103fee, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/9219a71272c34146bb9f29fd86fa9edf, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/bd9878d2f0444396b416003de988d47d] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=35.5 K 2024-11-20T10:25:17,892 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:17,892 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting eedbfa1b4d27400da6b25a605e103fee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732098314877 2024-11-20T10:25:17,892 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/A is initiating minor compaction (all files) 2024-11-20T10:25:17,892 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/A in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:17,892 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/60890690a10c4bcdaae82bbcc03f9815, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/dc691a7ee16f45a994af84be47f9b2d4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/9e93adcc2e0f4cdd8169ac57bfb10ba8] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=37.9 K 2024-11-20T10:25:17,892 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9219a71272c34146bb9f29fd86fa9edf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732098315518 2024-11-20T10:25:17,893 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting bd9878d2f0444396b416003de988d47d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732098317028 2024-11-20T10:25:17,893 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60890690a10c4bcdaae82bbcc03f9815, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732098314877 2024-11-20T10:25:17,893 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc691a7ee16f45a994af84be47f9b2d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732098315518 2024-11-20T10:25:17,893 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e93adcc2e0f4cdd8169ac57bfb10ba8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732098316706 2024-11-20T10:25:17,902 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:17,902 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:17,902 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/e69ef1e60eb744e2a168313a03ed4850 is 50, key is test_row_0/A:col10/1732098317029/Put/seqid=0 2024-11-20T10:25:17,903 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/0dc75f616b37489093bbb1bcec39bc9b is 50, key is test_row_0/B:col10/1732098317029/Put/seqid=0 2024-11-20T10:25:17,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742110_1286 (size=12493) 2024-11-20T10:25:17,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742111_1287 (size=12493) 2024-11-20T10:25:17,913 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/0dc75f616b37489093bbb1bcec39bc9b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0dc75f616b37489093bbb1bcec39bc9b 2024-11-20T10:25:17,920 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/B of 4030da3517ebc82c8a7cacbbe3bde6e8 into 0dc75f616b37489093bbb1bcec39bc9b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:17,920 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:17,920 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/B, priority=13, startTime=1732098317890; duration=0sec 2024-11-20T10:25:17,920 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:17,920 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:B 2024-11-20T10:25:17,921 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:17,922 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:17,922 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/C is initiating minor compaction (all files) 2024-11-20T10:25:17,922 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/C in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:17,923 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a05e2de6f46a40888bb1ebff57ca4f76, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2b7ef071bae84f25b342e781b1ff9924, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a3afd8bd973b458cbe6576f3a08ef27a] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=35.5 K 2024-11-20T10:25:17,924 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a05e2de6f46a40888bb1ebff57ca4f76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732098314877 2024-11-20T10:25:17,924 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b7ef071bae84f25b342e781b1ff9924, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732098315518 2024-11-20T10:25:17,924 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a3afd8bd973b458cbe6576f3a08ef27a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732098317028 2024-11-20T10:25:17,933 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#C#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:17,933 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/59c076a8203d4efe90511a6fb9041182 is 50, key is test_row_0/C:col10/1732098317029/Put/seqid=0 2024-11-20T10:25:17,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742112_1288 (size=12493) 2024-11-20T10:25:17,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T10:25:17,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:17,955 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T10:25:17,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/0eaa35c429914684b28d0e8d05eea9cd is 50, key is test_row_0/A:col10/1732098317032/Put/seqid=0 2024-11-20T10:25:17,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742113_1289 (size=12151) 2024-11-20T10:25:17,966 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/0eaa35c429914684b28d0e8d05eea9cd 2024-11-20T10:25:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T10:25:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/0bfd67564d684c53849f71d65f884171 is 50, key is test_row_0/B:col10/1732098317032/Put/seqid=0 2024-11-20T10:25:17,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742114_1290 (size=12151) 2024-11-20T10:25:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:18,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:18,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098378205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:18,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098378206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098378207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:18,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T10:25:18,312 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/e69ef1e60eb744e2a168313a03ed4850 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/e69ef1e60eb744e2a168313a03ed4850 2024-11-20T10:25:18,316 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/A of 4030da3517ebc82c8a7cacbbe3bde6e8 into e69ef1e60eb744e2a168313a03ed4850(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:18,316 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:18,316 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/A, priority=13, startTime=1732098317890; duration=0sec 2024-11-20T10:25:18,316 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:18,317 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:A 2024-11-20T10:25:18,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098378317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098378317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:18,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098378317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:18,356 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/59c076a8203d4efe90511a6fb9041182 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/59c076a8203d4efe90511a6fb9041182 2024-11-20T10:25:18,360 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/C of 4030da3517ebc82c8a7cacbbe3bde6e8 into 59c076a8203d4efe90511a6fb9041182(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:18,360 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:18,360 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/C, priority=13, startTime=1732098317891; duration=0sec 2024-11-20T10:25:18,361 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:18,361 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:C 2024-11-20T10:25:18,378 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/0bfd67564d684c53849f71d65f884171 2024-11-20T10:25:18,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/10d283a8ff8c4a40906915167e720b24 is 50, key is test_row_0/C:col10/1732098317032/Put/seqid=0 2024-11-20T10:25:18,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742115_1291 (size=12151) 2024-11-20T10:25:18,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098378526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:18,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098378527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:18,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098378527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T10:25:18,788 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/10d283a8ff8c4a40906915167e720b24 2024-11-20T10:25:18,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/0eaa35c429914684b28d0e8d05eea9cd as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/0eaa35c429914684b28d0e8d05eea9cd 2024-11-20T10:25:18,796 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/0eaa35c429914684b28d0e8d05eea9cd, entries=150, sequenceid=170, filesize=11.9 K 2024-11-20T10:25:18,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/0bfd67564d684c53849f71d65f884171 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0bfd67564d684c53849f71d65f884171 2024-11-20T10:25:18,801 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0bfd67564d684c53849f71d65f884171, entries=150, sequenceid=170, filesize=11.9 K 2024-11-20T10:25:18,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/10d283a8ff8c4a40906915167e720b24 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/10d283a8ff8c4a40906915167e720b24 2024-11-20T10:25:18,806 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/10d283a8ff8c4a40906915167e720b24, entries=150, sequenceid=170, filesize=11.9 K 2024-11-20T10:25:18,807 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 852ms, sequenceid=170, compaction requested=false 2024-11-20T10:25:18,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:18,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:18,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-20T10:25:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-20T10:25:18,811 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-20T10:25:18,811 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1590 sec 2024-11-20T10:25:18,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.1640 sec 2024-11-20T10:25:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:18,839 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:18,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-20T10:25:18,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098378840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:18,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/69323ac3a7b143b9911edb393259c3a2 is 50, key is test_row_0/A:col10/1732098318839/Put/seqid=0 2024-11-20T10:25:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:18,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098378844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:18,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098378844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:18,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742116_1292 (size=16931) 2024-11-20T10:25:18,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098378956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:18,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098378956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:18,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098378957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:19,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098379166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:19,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098379166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098379166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:19,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/69323ac3a7b143b9911edb393259c3a2 2024-11-20T10:25:19,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/13f836b97082403a8717cebfe25c3db4 is 50, key is test_row_0/B:col10/1732098318839/Put/seqid=0 2024-11-20T10:25:19,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742117_1293 (size=12151) 2024-11-20T10:25:19,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098379477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098379477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:19,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098379477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:19,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/13f836b97082403a8717cebfe25c3db4 2024-11-20T10:25:19,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/26c2ef1124eb4a4e9048c59522b648c3 is 50, key is test_row_0/C:col10/1732098318839/Put/seqid=0 2024-11-20T10:25:19,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742118_1294 (size=12151) 2024-11-20T10:25:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T10:25:19,797 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:19,797 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-20T10:25:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T10:25:19,799 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:19,802 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:19,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T10:25:19,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T10:25:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:19,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:19,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:19,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:19,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098379987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:19,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:19,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098379987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:19,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098379987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:20,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/26c2ef1124eb4a4e9048c59522b648c3 2024-11-20T10:25:20,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T10:25:20,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:20,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:20,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:20,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:20,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:20,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/69323ac3a7b143b9911edb393259c3a2 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/69323ac3a7b143b9911edb393259c3a2 2024-11-20T10:25:20,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/69323ac3a7b143b9911edb393259c3a2, entries=250, sequenceid=203, filesize=16.5 K 2024-11-20T10:25:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T10:25:20,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/13f836b97082403a8717cebfe25c3db4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/13f836b97082403a8717cebfe25c3db4 2024-11-20T10:25:20,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/13f836b97082403a8717cebfe25c3db4, entries=150, sequenceid=203, filesize=11.9 K 2024-11-20T10:25:20,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/26c2ef1124eb4a4e9048c59522b648c3 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/26c2ef1124eb4a4e9048c59522b648c3 2024-11-20T10:25:20,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/26c2ef1124eb4a4e9048c59522b648c3, entries=150, sequenceid=203, filesize=11.9 K 2024-11-20T10:25:20,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 1287ms, sequenceid=203, compaction requested=true 2024-11-20T10:25:20,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:20,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:25:20,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:20,127 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:20,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:25:20,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:20,127 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:20,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:25:20,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:20,128 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:20,128 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/B is initiating minor compaction (all files) 2024-11-20T10:25:20,128 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:20,128 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/B in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:20,128 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/A is initiating minor compaction (all files) 2024-11-20T10:25:20,128 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/A in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:20,128 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0dc75f616b37489093bbb1bcec39bc9b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0bfd67564d684c53849f71d65f884171, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/13f836b97082403a8717cebfe25c3db4] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=35.9 K 2024-11-20T10:25:20,128 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/e69ef1e60eb744e2a168313a03ed4850, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/0eaa35c429914684b28d0e8d05eea9cd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/69323ac3a7b143b9911edb393259c3a2] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=40.6 K 2024-11-20T10:25:20,128 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e69ef1e60eb744e2a168313a03ed4850, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732098317028 2024-11-20T10:25:20,128 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dc75f616b37489093bbb1bcec39bc9b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732098317028 2024-11-20T10:25:20,129 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eaa35c429914684b28d0e8d05eea9cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732098317032 2024-11-20T10:25:20,129 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bfd67564d684c53849f71d65f884171, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732098317032 2024-11-20T10:25:20,129 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69323ac3a7b143b9911edb393259c3a2, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732098318203 2024-11-20T10:25:20,129 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 13f836b97082403a8717cebfe25c3db4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732098318203 2024-11-20T10:25:20,135 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#B#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:20,136 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/2b385cd4bb1a433e9c81dc29bf540a5b is 50, key is test_row_0/B:col10/1732098318839/Put/seqid=0 2024-11-20T10:25:20,142 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#A#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:20,143 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/3b4bf050304f49db9189800743f5a058 is 50, key is test_row_0/A:col10/1732098318839/Put/seqid=0 2024-11-20T10:25:20,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742120_1296 (size=12595) 2024-11-20T10:25:20,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742119_1295 (size=12595) 2024-11-20T10:25:20,154 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/2b385cd4bb1a433e9c81dc29bf540a5b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/2b385cd4bb1a433e9c81dc29bf540a5b 2024-11-20T10:25:20,154 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/3b4bf050304f49db9189800743f5a058 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/3b4bf050304f49db9189800743f5a058 2024-11-20T10:25:20,160 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/A of 4030da3517ebc82c8a7cacbbe3bde6e8 into 3b4bf050304f49db9189800743f5a058(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:20,161 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:20,161 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/A, priority=13, startTime=1732098320127; duration=0sec 2024-11-20T10:25:20,161 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:20,161 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:A 2024-11-20T10:25:20,161 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:20,161 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/B of 4030da3517ebc82c8a7cacbbe3bde6e8 into 2b385cd4bb1a433e9c81dc29bf540a5b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:20,161 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:20,161 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/B, priority=13, startTime=1732098320127; duration=0sec 2024-11-20T10:25:20,161 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:20,161 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:B 2024-11-20T10:25:20,162 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:20,162 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/C is initiating minor compaction (all files) 2024-11-20T10:25:20,162 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/C in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:20,163 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/59c076a8203d4efe90511a6fb9041182, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/10d283a8ff8c4a40906915167e720b24, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/26c2ef1124eb4a4e9048c59522b648c3] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=35.9 K 2024-11-20T10:25:20,163 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59c076a8203d4efe90511a6fb9041182, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732098317028 2024-11-20T10:25:20,163 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10d283a8ff8c4a40906915167e720b24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732098317032 2024-11-20T10:25:20,164 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26c2ef1124eb4a4e9048c59522b648c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732098318203 2024-11-20T10:25:20,174 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#C#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:20,175 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/8c4830c86c2f45dbb5f86b0ce2df83eb is 50, key is test_row_0/C:col10/1732098318839/Put/seqid=0 2024-11-20T10:25:20,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742121_1297 (size=12595) 2024-11-20T10:25:20,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T10:25:20,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:20,260 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-20T10:25:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/17d7e3d78add477f9556af80451dd347 is 50, key is test_row_0/A:col10/1732098318842/Put/seqid=0 2024-11-20T10:25:20,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742122_1298 (size=7365) 2024-11-20T10:25:20,269 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/17d7e3d78add477f9556af80451dd347 2024-11-20T10:25:20,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/ad1f558499a846d88815cb062d0b7c72 is 50, key is test_row_0/B:col10/1732098318842/Put/seqid=0 2024-11-20T10:25:20,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742123_1299 (size=7365) 2024-11-20T10:25:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T10:25:20,589 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/8c4830c86c2f45dbb5f86b0ce2df83eb as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/8c4830c86c2f45dbb5f86b0ce2df83eb 2024-11-20T10:25:20,594 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/C of 4030da3517ebc82c8a7cacbbe3bde6e8 into 8c4830c86c2f45dbb5f86b0ce2df83eb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:20,594 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:20,594 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/C, priority=13, startTime=1732098320127; duration=0sec 2024-11-20T10:25:20,595 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:20,595 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:C 2024-11-20T10:25:20,679 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/ad1f558499a846d88815cb062d0b7c72 2024-11-20T10:25:20,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/f086b65dfc7c487eb65529018b2d75b6 is 50, key is test_row_0/C:col10/1732098318842/Put/seqid=0 2024-11-20T10:25:20,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742124_1300 (size=7365) 2024-11-20T10:25:20,694 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/f086b65dfc7c487eb65529018b2d75b6 2024-11-20T10:25:20,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/17d7e3d78add477f9556af80451dd347 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/17d7e3d78add477f9556af80451dd347 2024-11-20T10:25:20,710 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/17d7e3d78add477f9556af80451dd347, entries=50, sequenceid=210, filesize=7.2 K 2024-11-20T10:25:20,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/ad1f558499a846d88815cb062d0b7c72 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/ad1f558499a846d88815cb062d0b7c72 2024-11-20T10:25:20,715 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/ad1f558499a846d88815cb062d0b7c72, entries=50, sequenceid=210, filesize=7.2 K 2024-11-20T10:25:20,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/f086b65dfc7c487eb65529018b2d75b6 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/f086b65dfc7c487eb65529018b2d75b6 2024-11-20T10:25:20,721 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/f086b65dfc7c487eb65529018b2d75b6, entries=50, sequenceid=210, filesize=7.2 K 2024-11-20T10:25:20,722 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 462ms, sequenceid=210, compaction requested=false 2024-11-20T10:25:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-20T10:25:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-20T10:25:20,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-20T10:25:20,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 921 msec 2024-11-20T10:25:20,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 928 msec 2024-11-20T10:25:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T10:25:20,937 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:20,937 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-20T10:25:20,940 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T10:25:20,941 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:20,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:21,023 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:21,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:25:21,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/01798f701dd6485e9ccf79fc9bece3cc is 50, key is test_row_1/A:col10/1732098321022/Put/seqid=0 2024-11-20T10:25:21,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742125_1301 (size=12147) 2024-11-20T10:25:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T10:25:21,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098381075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:21,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098381075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:21,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098381076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:21,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T10:25:21,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:21,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:21,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:21,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098381187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:21,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:21,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098381187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:21,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:21,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098381188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:21,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:21,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:21,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T10:25:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:21,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T10:25:21,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098381397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:21,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098381397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:21,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098381397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:21,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T10:25:21,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:21,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:21,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:21,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/01798f701dd6485e9ccf79fc9bece3cc 2024-11-20T10:25:21,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/0ee6c4ac9de1448f958d163084afceb5 is 50, key is test_row_1/B:col10/1732098321022/Put/seqid=0 2024-11-20T10:25:21,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742126_1302 (size=9757) 2024-11-20T10:25:21,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T10:25:21,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:21,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:21,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:21,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T10:25:21,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098381706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:21,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098381707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:21,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098381707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:21,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:21,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:21,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T10:25:21,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:21,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:21,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T10:25:21,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:21,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:21,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:21,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:21,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/0ee6c4ac9de1448f958d163084afceb5 2024-11-20T10:25:21,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/467b6fa392084ab3b9e234ce98ae4cc1 is 50, key is test_row_1/C:col10/1732098321022/Put/seqid=0 2024-11-20T10:25:21,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742127_1303 (size=9757) 2024-11-20T10:25:22,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T10:25:22,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:22,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:22,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:22,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:22,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T10:25:22,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T10:25:22,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:22,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:22,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:22,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:22,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:22,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098382216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:22,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098382216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:22,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098382216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:22,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T10:25:22,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:22,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:22,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:22,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:22,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:22,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/467b6fa392084ab3b9e234ce98ae4cc1 2024-11-20T10:25:22,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/01798f701dd6485e9ccf79fc9bece3cc as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/01798f701dd6485e9ccf79fc9bece3cc 2024-11-20T10:25:22,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/01798f701dd6485e9ccf79fc9bece3cc, entries=150, sequenceid=222, filesize=11.9 K 2024-11-20T10:25:22,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/0ee6c4ac9de1448f958d163084afceb5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0ee6c4ac9de1448f958d163084afceb5 2024-11-20T10:25:22,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0ee6c4ac9de1448f958d163084afceb5, entries=100, sequenceid=222, filesize=9.5 K 2024-11-20T10:25:22,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/467b6fa392084ab3b9e234ce98ae4cc1 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/467b6fa392084ab3b9e234ce98ae4cc1 2024-11-20T10:25:22,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/467b6fa392084ab3b9e234ce98ae4cc1, entries=100, sequenceid=222, filesize=9.5 K 2024-11-20T10:25:22,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 1405ms, sequenceid=222, compaction requested=true 2024-11-20T10:25:22,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:22,429 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:25:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:22,430 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:22,430 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:22,430 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/A is initiating minor compaction (all files) 2024-11-20T10:25:22,430 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/A in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:22,430 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/3b4bf050304f49db9189800743f5a058, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/17d7e3d78add477f9556af80451dd347, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/01798f701dd6485e9ccf79fc9bece3cc] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=31.4 K 2024-11-20T10:25:22,431 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b4bf050304f49db9189800743f5a058, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732098318203 2024-11-20T10:25:22,432 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 29717 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:22,432 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/B is initiating minor compaction (all files) 2024-11-20T10:25:22,432 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/B in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:22,432 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/2b385cd4bb1a433e9c81dc29bf540a5b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/ad1f558499a846d88815cb062d0b7c72, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0ee6c4ac9de1448f958d163084afceb5] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=29.0 K 2024-11-20T10:25:22,432 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17d7e3d78add477f9556af80451dd347, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732098318842 2024-11-20T10:25:22,432 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b385cd4bb1a433e9c81dc29bf540a5b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732098318203 2024-11-20T10:25:22,432 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01798f701dd6485e9ccf79fc9bece3cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732098321013 2024-11-20T10:25:22,433 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ad1f558499a846d88815cb062d0b7c72, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732098318842 2024-11-20T10:25:22,434 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ee6c4ac9de1448f958d163084afceb5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732098321013 2024-11-20T10:25:22,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:25:22,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:22,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:25:22,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:22,455 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#A#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:22,456 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/a84a80e39a8e4f2c8908a9e05e33aa3f is 50, key is test_row_0/A:col10/1732098318842/Put/seqid=0 2024-11-20T10:25:22,460 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#B#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:22,461 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/93ed8f04d3e24623a6b0061a30df3007 is 50, key is test_row_0/B:col10/1732098318842/Put/seqid=0 2024-11-20T10:25:22,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T10:25:22,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:22,482 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T10:25:22,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742128_1304 (size=12697) 2024-11-20T10:25:22,494 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/a84a80e39a8e4f2c8908a9e05e33aa3f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/a84a80e39a8e4f2c8908a9e05e33aa3f 2024-11-20T10:25:22,502 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/A of 4030da3517ebc82c8a7cacbbe3bde6e8 into a84a80e39a8e4f2c8908a9e05e33aa3f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:22,502 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:22,502 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/A, priority=13, startTime=1732098322428; duration=0sec 2024-11-20T10:25:22,502 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:22,502 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:A 2024-11-20T10:25:22,502 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:22,503 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 29717 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:22,503 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/C is initiating minor compaction (all files) 2024-11-20T10:25:22,503 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/C in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:22,503 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/8c4830c86c2f45dbb5f86b0ce2df83eb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/f086b65dfc7c487eb65529018b2d75b6, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/467b6fa392084ab3b9e234ce98ae4cc1] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=29.0 K 2024-11-20T10:25:22,504 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c4830c86c2f45dbb5f86b0ce2df83eb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732098318203 2024-11-20T10:25:22,505 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f086b65dfc7c487eb65529018b2d75b6, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732098318842 2024-11-20T10:25:22,505 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 467b6fa392084ab3b9e234ce98ae4cc1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732098321013 2024-11-20T10:25:22,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742129_1305 (size=12697) 2024-11-20T10:25:22,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/018859dd5fbd4d48ba8ed4cccf7cc1d1 is 50, key is test_row_0/A:col10/1732098321068/Put/seqid=0 2024-11-20T10:25:22,521 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/93ed8f04d3e24623a6b0061a30df3007 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/93ed8f04d3e24623a6b0061a30df3007 2024-11-20T10:25:22,527 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/B of 4030da3517ebc82c8a7cacbbe3bde6e8 into 93ed8f04d3e24623a6b0061a30df3007(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:22,527 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:22,528 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/B, priority=13, startTime=1732098322430; duration=0sec 2024-11-20T10:25:22,528 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:22,528 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:B 2024-11-20T10:25:22,542 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#C#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:22,543 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/65ef6a899e4f4e65bd9d3ee6a4d572ba is 50, key is test_row_0/C:col10/1732098318842/Put/seqid=0 2024-11-20T10:25:22,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742130_1306 (size=12151) 2024-11-20T10:25:22,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742131_1307 (size=12697) 2024-11-20T10:25:22,585 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/65ef6a899e4f4e65bd9d3ee6a4d572ba as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/65ef6a899e4f4e65bd9d3ee6a4d572ba 2024-11-20T10:25:22,595 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/C of 4030da3517ebc82c8a7cacbbe3bde6e8 into 65ef6a899e4f4e65bd9d3ee6a4d572ba(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:22,595 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:22,595 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/C, priority=13, startTime=1732098322445; duration=0sec 2024-11-20T10:25:22,595 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:22,595 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:C 2024-11-20T10:25:22,950 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/018859dd5fbd4d48ba8ed4cccf7cc1d1 2024-11-20T10:25:22,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/edf4f0d3a81445aaa8c87c847589b592 is 50, key is test_row_0/B:col10/1732098321068/Put/seqid=0 2024-11-20T10:25:22,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742132_1308 (size=12151) 2024-11-20T10:25:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T10:25:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:23,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:23,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098383243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:23,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098383244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:23,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098383246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:23,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098383357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:23,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098383357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:23,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098383357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:23,374 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/edf4f0d3a81445aaa8c87c847589b592 2024-11-20T10:25:23,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/e5931e4ee95240d8922f812123d0b43e is 50, key is test_row_0/C:col10/1732098321068/Put/seqid=0 2024-11-20T10:25:23,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742133_1309 (size=12151) 2024-11-20T10:25:23,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098383567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:23,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098383567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:23,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098383567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:23,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:23,787 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/e5931e4ee95240d8922f812123d0b43e 2024-11-20T10:25:23,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/018859dd5fbd4d48ba8ed4cccf7cc1d1 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/018859dd5fbd4d48ba8ed4cccf7cc1d1 2024-11-20T10:25:23,805 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/018859dd5fbd4d48ba8ed4cccf7cc1d1, entries=150, sequenceid=247, filesize=11.9 K 2024-11-20T10:25:23,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/edf4f0d3a81445aaa8c87c847589b592 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/edf4f0d3a81445aaa8c87c847589b592 2024-11-20T10:25:23,810 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/edf4f0d3a81445aaa8c87c847589b592, entries=150, sequenceid=247, filesize=11.9 K 2024-11-20T10:25:23,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/e5931e4ee95240d8922f812123d0b43e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/e5931e4ee95240d8922f812123d0b43e 2024-11-20T10:25:23,819 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/e5931e4ee95240d8922f812123d0b43e, entries=150, sequenceid=247, filesize=11.9 K 2024-11-20T10:25:23,820 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 1337ms, sequenceid=247, compaction requested=false 2024-11-20T10:25:23,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:23,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:23,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-20T10:25:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-20T10:25:23,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-20T10:25:23,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8800 sec 2024-11-20T10:25:23,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.8890 sec 2024-11-20T10:25:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:23,879 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:23,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T10:25:23,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/533a7426f809449790ec26ad8e963531 is 50, key is test_row_0/A:col10/1732098323879/Put/seqid=0 2024-11-20T10:25:23,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742134_1310 (size=12301) 2024-11-20T10:25:23,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098383949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:23,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098383949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:23,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098383949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:24,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098384056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:24,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:24,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098384057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:24,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:24,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098384067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:24,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098384266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:24,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098384266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:24,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098384277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:24,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/533a7426f809449790ec26ad8e963531 2024-11-20T10:25:24,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/8157856aa10f4695833dc8ece1a3ddb0 is 50, key is test_row_0/B:col10/1732098323879/Put/seqid=0 2024-11-20T10:25:24,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742135_1311 (size=12301) 2024-11-20T10:25:24,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098384577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:24,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098384577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:24,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098384586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:24,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/8157856aa10f4695833dc8ece1a3ddb0 2024-11-20T10:25:24,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/408f794c1106466a84d9d012921e96fd is 50, key is test_row_0/C:col10/1732098323879/Put/seqid=0 2024-11-20T10:25:24,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742136_1312 (size=12301) 2024-11-20T10:25:25,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098385087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:25,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098385087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T10:25:25,097 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:25,097 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:25,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098385096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:25,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:25,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:25,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-20T10:25:25,100 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T10:25:25,101 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:25,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:25,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/408f794c1106466a84d9d012921e96fd 2024-11-20T10:25:25,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/533a7426f809449790ec26ad8e963531 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/533a7426f809449790ec26ad8e963531 2024-11-20T10:25:25,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/533a7426f809449790ec26ad8e963531, entries=150, sequenceid=263, filesize=12.0 K 2024-11-20T10:25:25,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/8157856aa10f4695833dc8ece1a3ddb0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8157856aa10f4695833dc8ece1a3ddb0 2024-11-20T10:25:25,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8157856aa10f4695833dc8ece1a3ddb0, entries=150, sequenceid=263, filesize=12.0 K 2024-11-20T10:25:25,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/408f794c1106466a84d9d012921e96fd as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/408f794c1106466a84d9d012921e96fd 2024-11-20T10:25:25,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/408f794c1106466a84d9d012921e96fd, entries=150, sequenceid=263, filesize=12.0 K 2024-11-20T10:25:25,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 1257ms, sequenceid=263, compaction requested=true 2024-11-20T10:25:25,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:25,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:25:25,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:25,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:25:25,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:25,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:25:25,137 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:25,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T10:25:25,137 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:25,139 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:25,139 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/A is initiating minor compaction (all files) 2024-11-20T10:25:25,139 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/A in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:25,139 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/a84a80e39a8e4f2c8908a9e05e33aa3f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/018859dd5fbd4d48ba8ed4cccf7cc1d1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/533a7426f809449790ec26ad8e963531] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=36.3 K 2024-11-20T10:25:25,139 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:25,140 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/B is initiating minor compaction (all files) 2024-11-20T10:25:25,140 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a84a80e39a8e4f2c8908a9e05e33aa3f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732098318842 2024-11-20T10:25:25,140 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/B in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:25,140 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/93ed8f04d3e24623a6b0061a30df3007, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/edf4f0d3a81445aaa8c87c847589b592, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8157856aa10f4695833dc8ece1a3ddb0] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=36.3 K 2024-11-20T10:25:25,140 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93ed8f04d3e24623a6b0061a30df3007, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732098318842 2024-11-20T10:25:25,140 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 018859dd5fbd4d48ba8ed4cccf7cc1d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732098321068 2024-11-20T10:25:25,140 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting edf4f0d3a81445aaa8c87c847589b592, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732098321068 2024-11-20T10:25:25,141 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 533a7426f809449790ec26ad8e963531, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732098323243 2024-11-20T10:25:25,141 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8157856aa10f4695833dc8ece1a3ddb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732098323243 2024-11-20T10:25:25,151 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#B#compaction#264 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:25,151 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/d92e8f0519a5448cab01a37b09eb28b1 is 50, key is test_row_0/B:col10/1732098323879/Put/seqid=0 2024-11-20T10:25:25,153 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#A#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:25,154 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/4b82bd2c75d44cb795f86d6581c99e09 is 50, key is test_row_0/A:col10/1732098323879/Put/seqid=0 2024-11-20T10:25:25,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742137_1313 (size=12949) 2024-11-20T10:25:25,167 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/d92e8f0519a5448cab01a37b09eb28b1 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/d92e8f0519a5448cab01a37b09eb28b1 2024-11-20T10:25:25,181 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/B of 4030da3517ebc82c8a7cacbbe3bde6e8 into d92e8f0519a5448cab01a37b09eb28b1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:25,181 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:25,181 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/B, priority=13, startTime=1732098325137; duration=0sec 2024-11-20T10:25:25,181 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:25,182 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:B 2024-11-20T10:25:25,182 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:25,183 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:25,183 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/C is initiating minor compaction (all files) 2024-11-20T10:25:25,183 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/C in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:25,183 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/65ef6a899e4f4e65bd9d3ee6a4d572ba, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/e5931e4ee95240d8922f812123d0b43e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/408f794c1106466a84d9d012921e96fd] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=36.3 K 2024-11-20T10:25:25,185 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65ef6a899e4f4e65bd9d3ee6a4d572ba, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732098318842 2024-11-20T10:25:25,187 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5931e4ee95240d8922f812123d0b43e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732098321068 2024-11-20T10:25:25,188 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 408f794c1106466a84d9d012921e96fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732098323243 2024-11-20T10:25:25,206 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#C#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:25,206 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/2de6de468ae34d098a41d504bde6f3e2 is 50, key is test_row_0/C:col10/1732098323879/Put/seqid=0 2024-11-20T10:25:25,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T10:25:25,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742138_1314 (size=12949) 2024-11-20T10:25:25,216 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/4b82bd2c75d44cb795f86d6581c99e09 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/4b82bd2c75d44cb795f86d6581c99e09 2024-11-20T10:25:25,223 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/A of 4030da3517ebc82c8a7cacbbe3bde6e8 into 4b82bd2c75d44cb795f86d6581c99e09(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:25,223 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:25,223 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/A, priority=13, startTime=1732098325137; duration=0sec 2024-11-20T10:25:25,223 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:25,223 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:A 2024-11-20T10:25:25,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742139_1315 (size=12949) 2024-11-20T10:25:25,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T10:25:25,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:25,254 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T10:25:25,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/304e4e9192cf4ee0a32b158490df0614 is 50, key is test_row_0/A:col10/1732098323923/Put/seqid=0 2024-11-20T10:25:25,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742140_1316 (size=12301) 2024-11-20T10:25:25,318 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/304e4e9192cf4ee0a32b158490df0614 2024-11-20T10:25:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/19da843c349c404e8af24bd9bc399717 is 50, key is test_row_0/B:col10/1732098323923/Put/seqid=0 2024-11-20T10:25:25,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742141_1317 (size=12301) 2024-11-20T10:25:25,378 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/19da843c349c404e8af24bd9bc399717 2024-11-20T10:25:25,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/962febe646954f29adfd2af2f317804b is 50, key is test_row_0/C:col10/1732098323923/Put/seqid=0 2024-11-20T10:25:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T10:25:25,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742142_1318 (size=12301) 2024-11-20T10:25:25,630 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/2de6de468ae34d098a41d504bde6f3e2 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2de6de468ae34d098a41d504bde6f3e2 2024-11-20T10:25:25,634 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/C of 4030da3517ebc82c8a7cacbbe3bde6e8 into 2de6de468ae34d098a41d504bde6f3e2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:25,635 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:25,635 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/C, priority=13, startTime=1732098325137; duration=0sec 2024-11-20T10:25:25,635 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:25,635 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:C 2024-11-20T10:25:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T10:25:25,836 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/962febe646954f29adfd2af2f317804b 2024-11-20T10:25:25,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/304e4e9192cf4ee0a32b158490df0614 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/304e4e9192cf4ee0a32b158490df0614 2024-11-20T10:25:25,846 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/304e4e9192cf4ee0a32b158490df0614, entries=150, sequenceid=289, filesize=12.0 K 2024-11-20T10:25:25,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/19da843c349c404e8af24bd9bc399717 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/19da843c349c404e8af24bd9bc399717 2024-11-20T10:25:25,851 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/19da843c349c404e8af24bd9bc399717, entries=150, sequenceid=289, filesize=12.0 K 2024-11-20T10:25:25,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/962febe646954f29adfd2af2f317804b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/962febe646954f29adfd2af2f317804b 2024-11-20T10:25:25,855 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/962febe646954f29adfd2af2f317804b, entries=150, sequenceid=289, filesize=12.0 K 2024-11-20T10:25:25,856 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 602ms, sequenceid=289, compaction requested=false 2024-11-20T10:25:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-20T10:25:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-20T10:25:25,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-20T10:25:25,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 757 msec 2024-11-20T10:25:25,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 763 msec 2024-11-20T10:25:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:26,115 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:26,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:25:26,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/82ff0b73b5f5452fa9a87b3eab236c58 is 50, key is test_row_0/A:col10/1732098326114/Put/seqid=0 2024-11-20T10:25:26,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742143_1319 (size=12301) 2024-11-20T10:25:26,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098386187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:26,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098386188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:26,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:26,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:26,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098386188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:26,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T10:25:26,237 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:26,237 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-20T10:25:26,239 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T10:25:26,240 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:26,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:26,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098386296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:26,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:26,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098386296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:26,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098386296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T10:25:26,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T10:25:26,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:26,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:26,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:26,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:26,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:26,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098386507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098386507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098386507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:26,535 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T10:25:26,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T10:25:26,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:26,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:26,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:26,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:26,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:26,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/82ff0b73b5f5452fa9a87b3eab236c58 2024-11-20T10:25:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:26,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/f2ffe30ed6c14aeda390aa80e2b84ecf is 50, key is test_row_0/B:col10/1732098326114/Put/seqid=0 2024-11-20T10:25:26,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742144_1320 (size=12301) 2024-11-20T10:25:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T10:25:26,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/f2ffe30ed6c14aeda390aa80e2b84ecf 2024-11-20T10:25:26,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/04b6eed9ffb243c08a68ee3b269f8e16 is 50, key is test_row_0/C:col10/1732098326114/Put/seqid=0 2024-11-20T10:25:26,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742145_1321 (size=12301) 2024-11-20T10:25:26,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/04b6eed9ffb243c08a68ee3b269f8e16 2024-11-20T10:25:26,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/82ff0b73b5f5452fa9a87b3eab236c58 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/82ff0b73b5f5452fa9a87b3eab236c58 2024-11-20T10:25:26,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/82ff0b73b5f5452fa9a87b3eab236c58, entries=150, sequenceid=302, filesize=12.0 K 2024-11-20T10:25:26,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/f2ffe30ed6c14aeda390aa80e2b84ecf as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/f2ffe30ed6c14aeda390aa80e2b84ecf 2024-11-20T10:25:26,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/f2ffe30ed6c14aeda390aa80e2b84ecf, entries=150, sequenceid=302, filesize=12.0 K 2024-11-20T10:25:26,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/04b6eed9ffb243c08a68ee3b269f8e16 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/04b6eed9ffb243c08a68ee3b269f8e16 2024-11-20T10:25:26,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/04b6eed9ffb243c08a68ee3b269f8e16, entries=150, sequenceid=302, filesize=12.0 K 2024-11-20T10:25:26,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 480ms, sequenceid=302, compaction requested=true 2024-11-20T10:25:26,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:26,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:25:26,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:26,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:25:26,595 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:26,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:26,595 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:26,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:25:26,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:26,597 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:26,597 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/B is initiating minor compaction (all files) 2024-11-20T10:25:26,597 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/B in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:26,597 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/d92e8f0519a5448cab01a37b09eb28b1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/19da843c349c404e8af24bd9bc399717, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/f2ffe30ed6c14aeda390aa80e2b84ecf] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=36.7 K 2024-11-20T10:25:26,597 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:26,597 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/A is initiating minor compaction (all files) 2024-11-20T10:25:26,597 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/A in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:26,598 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/4b82bd2c75d44cb795f86d6581c99e09, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/304e4e9192cf4ee0a32b158490df0614, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/82ff0b73b5f5452fa9a87b3eab236c58] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=36.7 K 2024-11-20T10:25:26,598 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d92e8f0519a5448cab01a37b09eb28b1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732098323243 2024-11-20T10:25:26,598 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b82bd2c75d44cb795f86d6581c99e09, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732098323243 2024-11-20T10:25:26,598 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 19da843c349c404e8af24bd9bc399717, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732098323911 2024-11-20T10:25:26,598 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 304e4e9192cf4ee0a32b158490df0614, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732098323911 2024-11-20T10:25:26,598 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f2ffe30ed6c14aeda390aa80e2b84ecf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732098326112 2024-11-20T10:25:26,598 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82ff0b73b5f5452fa9a87b3eab236c58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732098326112 2024-11-20T10:25:26,605 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#B#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:26,605 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/c14329e6590b481cb1f5ece31116b717 is 50, key is test_row_0/B:col10/1732098326114/Put/seqid=0 2024-11-20T10:25:26,606 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#A#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:26,606 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/80839fa25c9f4b64b95c124849cc522e is 50, key is test_row_0/A:col10/1732098326114/Put/seqid=0 2024-11-20T10:25:26,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742146_1322 (size=13051) 2024-11-20T10:25:26,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742147_1323 (size=13051) 2024-11-20T10:25:26,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T10:25:26,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:26,699 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T10:25:26,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/b85b81f3a8cb41db8c0747fbefc8386b is 50, key is test_row_0/A:col10/1732098326150/Put/seqid=0 2024-11-20T10:25:26,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742148_1324 (size=12301) 2024-11-20T10:25:26,723 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/b85b81f3a8cb41db8c0747fbefc8386b 2024-11-20T10:25:26,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/8e9ddc37eb26425c91169899fe15840b is 50, key is test_row_0/B:col10/1732098326150/Put/seqid=0 2024-11-20T10:25:26,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742149_1325 (size=12301) 2024-11-20T10:25:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:26,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:26,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098386830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:26,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098386831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:26,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098386834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:26,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T10:25:26,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098386947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:26,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098386947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098386947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,022 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/c14329e6590b481cb1f5ece31116b717 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/c14329e6590b481cb1f5ece31116b717 2024-11-20T10:25:27,026 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/B of 4030da3517ebc82c8a7cacbbe3bde6e8 into c14329e6590b481cb1f5ece31116b717(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:27,026 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:27,026 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/B, priority=13, startTime=1732098326595; duration=0sec 2024-11-20T10:25:27,026 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:27,026 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:B 2024-11-20T10:25:27,027 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:27,028 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:27,028 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/C is initiating minor compaction (all files) 2024-11-20T10:25:27,028 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/C in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:27,028 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2de6de468ae34d098a41d504bde6f3e2, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/962febe646954f29adfd2af2f317804b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/04b6eed9ffb243c08a68ee3b269f8e16] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=36.7 K 2024-11-20T10:25:27,029 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2de6de468ae34d098a41d504bde6f3e2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732098323243 2024-11-20T10:25:27,029 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/80839fa25c9f4b64b95c124849cc522e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/80839fa25c9f4b64b95c124849cc522e 2024-11-20T10:25:27,029 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 962febe646954f29adfd2af2f317804b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732098323911 2024-11-20T10:25:27,029 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b6eed9ffb243c08a68ee3b269f8e16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732098326112 2024-11-20T10:25:27,033 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/A of 4030da3517ebc82c8a7cacbbe3bde6e8 into 80839fa25c9f4b64b95c124849cc522e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:27,033 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:27,033 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/A, priority=13, startTime=1732098326595; duration=0sec 2024-11-20T10:25:27,033 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:27,033 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:A 2024-11-20T10:25:27,035 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#C#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:27,036 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/a67ffbe9b6514357a8b3c89f92033348 is 50, key is test_row_0/C:col10/1732098326114/Put/seqid=0 2024-11-20T10:25:27,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742150_1326 (size=13051) 2024-11-20T10:25:27,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098387157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098387157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098387157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,178 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/8e9ddc37eb26425c91169899fe15840b 2024-11-20T10:25:27,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/2c2648d8cb964dc3851d698bc70ab34f is 50, key is test_row_0/C:col10/1732098326150/Put/seqid=0 2024-11-20T10:25:27,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742151_1327 (size=12301) 2024-11-20T10:25:27,192 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/2c2648d8cb964dc3851d698bc70ab34f 2024-11-20T10:25:27,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/b85b81f3a8cb41db8c0747fbefc8386b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/b85b81f3a8cb41db8c0747fbefc8386b 2024-11-20T10:25:27,200 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/b85b81f3a8cb41db8c0747fbefc8386b, entries=150, sequenceid=326, filesize=12.0 K 2024-11-20T10:25:27,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/8e9ddc37eb26425c91169899fe15840b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8e9ddc37eb26425c91169899fe15840b 2024-11-20T10:25:27,205 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8e9ddc37eb26425c91169899fe15840b, entries=150, sequenceid=326, filesize=12.0 K 2024-11-20T10:25:27,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/2c2648d8cb964dc3851d698bc70ab34f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2c2648d8cb964dc3851d698bc70ab34f 2024-11-20T10:25:27,210 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2c2648d8cb964dc3851d698bc70ab34f, entries=150, sequenceid=326, filesize=12.0 K 2024-11-20T10:25:27,211 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 512ms, sequenceid=326, compaction requested=false 2024-11-20T10:25:27,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:27,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:27,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-20T10:25:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-20T10:25:27,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-20T10:25:27,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 972 msec 2024-11-20T10:25:27,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 977 msec 2024-11-20T10:25:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T10:25:27,377 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:27,377 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-20T10:25:27,380 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T10:25:27,381 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:27,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:27,446 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/a67ffbe9b6514357a8b3c89f92033348 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a67ffbe9b6514357a8b3c89f92033348 2024-11-20T10:25:27,456 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/C of 4030da3517ebc82c8a7cacbbe3bde6e8 into a67ffbe9b6514357a8b3c89f92033348(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:27,456 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:27,456 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/C, priority=13, startTime=1732098326595; duration=0sec 2024-11-20T10:25:27,456 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:27,456 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:C 2024-11-20T10:25:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:27,469 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:27,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T10:25:27,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/f23b93c08257427d850e43142329729f is 50, key is test_row_0/A:col10/1732098326824/Put/seqid=0 2024-11-20T10:25:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T10:25:27,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742152_1328 (size=14741) 2024-11-20T10:25:27,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098387508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098387509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098387511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1732098387511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T10:25:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:27,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:27,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41372 deadline: 1732098387537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,541 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in 4030da3517ebc82c8a7cacbbe3bde6e8 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18317 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:25:27,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098387616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098387616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1732098387627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098387627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T10:25:27,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:27,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:27,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:27,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:27,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T10:25:27,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098387827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098387827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098387837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1732098387837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:27,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T10:25:27,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:27,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:27,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:27,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:27,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:27,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/f23b93c08257427d850e43142329729f 2024-11-20T10:25:27,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/6052020b111d473ca2030941c0006e6d is 50, key is test_row_0/B:col10/1732098326824/Put/seqid=0 2024-11-20T10:25:27,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742153_1329 (size=12301) 2024-11-20T10:25:27,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T10:25:27,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:27,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:27,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:27,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:27,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T10:25:28,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098388137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:28,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098388137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:28,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T10:25:28,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:28,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:28,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:28,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098388147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:28,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1732098388147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:28,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T10:25:28,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:28,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:28,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:28,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/6052020b111d473ca2030941c0006e6d 2024-11-20T10:25:28,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/8bd25b038164432ea0296657eac7ebe6 is 50, key is test_row_0/C:col10/1732098326824/Put/seqid=0 2024-11-20T10:25:28,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742154_1330 (size=12301) 2024-11-20T10:25:28,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T10:25:28,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:28,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:28,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:28,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T10:25:28,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T10:25:28,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:28,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:28,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:28,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41354 deadline: 1732098388646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:28,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41400 deadline: 1732098388646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:28,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41414 deadline: 1732098388667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:28,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41412 deadline: 1732098388667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:25:28,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T10:25:28,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:28,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:28,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:28,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:28,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/8bd25b038164432ea0296657eac7ebe6 2024-11-20T10:25:28,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/f23b93c08257427d850e43142329729f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/f23b93c08257427d850e43142329729f 2024-11-20T10:25:28,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/f23b93c08257427d850e43142329729f, entries=200, sequenceid=343, filesize=14.4 K 2024-11-20T10:25:28,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/6052020b111d473ca2030941c0006e6d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/6052020b111d473ca2030941c0006e6d 2024-11-20T10:25:28,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/6052020b111d473ca2030941c0006e6d, entries=150, sequenceid=343, filesize=12.0 K 2024-11-20T10:25:28,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/8bd25b038164432ea0296657eac7ebe6 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/8bd25b038164432ea0296657eac7ebe6 2024-11-20T10:25:28,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/8bd25b038164432ea0296657eac7ebe6, entries=150, sequenceid=343, filesize=12.0 K 2024-11-20T10:25:28,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 1363ms, sequenceid=343, compaction requested=true 2024-11-20T10:25:28,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:28,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:25:28,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:28,833 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:28,833 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:28,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:25:28,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:28,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4030da3517ebc82c8a7cacbbe3bde6e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:25:28,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:28,834 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:28,834 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:28,834 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/A is initiating minor compaction (all files) 2024-11-20T10:25:28,834 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/B is initiating minor compaction (all files) 2024-11-20T10:25:28,834 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/B in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:28,834 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/A in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:28,834 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/80839fa25c9f4b64b95c124849cc522e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/b85b81f3a8cb41db8c0747fbefc8386b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/f23b93c08257427d850e43142329729f] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=39.2 K 2024-11-20T10:25:28,834 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/c14329e6590b481cb1f5ece31116b717, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8e9ddc37eb26425c91169899fe15840b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/6052020b111d473ca2030941c0006e6d] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=36.8 K 2024-11-20T10:25:28,835 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80839fa25c9f4b64b95c124849cc522e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732098326112 2024-11-20T10:25:28,835 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c14329e6590b481cb1f5ece31116b717, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732098326112 2024-11-20T10:25:28,835 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e9ddc37eb26425c91169899fe15840b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732098326146 2024-11-20T10:25:28,835 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting b85b81f3a8cb41db8c0747fbefc8386b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732098326146 2024-11-20T10:25:28,835 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f23b93c08257427d850e43142329729f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732098326824 2024-11-20T10:25:28,836 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 6052020b111d473ca2030941c0006e6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732098326824 2024-11-20T10:25:28,848 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#B#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:28,849 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/5dec1da6929b4f80b8364d3dfdc17287 is 50, key is test_row_0/B:col10/1732098326824/Put/seqid=0 2024-11-20T10:25:28,852 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#A#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:28,853 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/8099704e1f0c4dbca96f53b5acdcab1a is 50, key is test_row_0/A:col10/1732098326824/Put/seqid=0 2024-11-20T10:25:28,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742156_1332 (size=13153) 2024-11-20T10:25:28,910 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/8099704e1f0c4dbca96f53b5acdcab1a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/8099704e1f0c4dbca96f53b5acdcab1a 2024-11-20T10:25:28,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T10:25:28,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:28,918 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-20T10:25:28,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742155_1331 (size=13153) 2024-11-20T10:25:28,920 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/A of 4030da3517ebc82c8a7cacbbe3bde6e8 into 8099704e1f0c4dbca96f53b5acdcab1a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:28,920 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:28,920 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/A, priority=13, startTime=1732098328833; duration=0sec 2024-11-20T10:25:28,920 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:28,920 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:A 2024-11-20T10:25:28,920 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:28,923 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:28,923 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): 4030da3517ebc82c8a7cacbbe3bde6e8/C is initiating minor compaction (all files) 2024-11-20T10:25:28,923 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4030da3517ebc82c8a7cacbbe3bde6e8/C in TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:28,923 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a67ffbe9b6514357a8b3c89f92033348, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2c2648d8cb964dc3851d698bc70ab34f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/8bd25b038164432ea0296657eac7ebe6] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp, totalSize=36.8 K 2024-11-20T10:25:28,925 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a67ffbe9b6514357a8b3c89f92033348, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732098326112 2024-11-20T10:25:28,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/1fa2e19457ef42dba79e961625ee48ef is 50, key is test_row_0/A:col10/1732098327510/Put/seqid=0 2024-11-20T10:25:28,926 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/5dec1da6929b4f80b8364d3dfdc17287 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/5dec1da6929b4f80b8364d3dfdc17287 2024-11-20T10:25:28,926 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c2648d8cb964dc3851d698bc70ab34f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732098326146 2024-11-20T10:25:28,927 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bd25b038164432ea0296657eac7ebe6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732098326824 2024-11-20T10:25:28,932 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/B of 4030da3517ebc82c8a7cacbbe3bde6e8 into 5dec1da6929b4f80b8364d3dfdc17287(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:28,932 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:28,932 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/B, priority=13, startTime=1732098328833; duration=0sec 2024-11-20T10:25:28,932 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:28,932 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:B 2024-11-20T10:25:28,956 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4030da3517ebc82c8a7cacbbe3bde6e8#C#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:28,957 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/7ccd064387174497b29b85df492caca0 is 50, key is test_row_0/C:col10/1732098326824/Put/seqid=0 2024-11-20T10:25:29,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742157_1333 (size=12301) 2024-11-20T10:25:29,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742158_1334 (size=13153) 2024-11-20T10:25:29,040 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/7ccd064387174497b29b85df492caca0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/7ccd064387174497b29b85df492caca0 2024-11-20T10:25:29,046 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4030da3517ebc82c8a7cacbbe3bde6e8/C of 4030da3517ebc82c8a7cacbbe3bde6e8 into 7ccd064387174497b29b85df492caca0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:29,046 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:29,047 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., storeName=4030da3517ebc82c8a7cacbbe3bde6e8/C, priority=13, startTime=1732098328833; duration=0sec 2024-11-20T10:25:29,047 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:29,047 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4030da3517ebc82c8a7cacbbe3bde6e8:C 2024-11-20T10:25:29,189 INFO [Thread-1195 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1195. 2024-11-20T10:25:29,189 DEBUG [Thread-1195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:29,190 DEBUG [Thread-1195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,191 INFO [Thread-1199 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1199. 2024-11-20T10:25:29,191 INFO [Thread-1197 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1197. 2024-11-20T10:25:29,191 DEBUG [Thread-1197 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:29,192 DEBUG [Thread-1197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,192 DEBUG [Thread-1197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,192 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:29,192 INFO [Thread-1201 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1201. 2024-11-20T10:25:29,192 DEBUG [Thread-1201 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:29,192 DEBUG [Thread-1201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,192 DEBUG [Thread-1201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,192 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:29,192 DEBUG [Thread-1199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:29,192 DEBUG [Thread-1199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,192 DEBUG [Thread-1199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,192 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:29,193 DEBUG [Thread-1195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:29,193 INFO [Thread-1193 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1193. 2024-11-20T10:25:29,193 DEBUG [Thread-1193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:29,193 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,193 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:29,414 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/1fa2e19457ef42dba79e961625ee48ef 2024-11-20T10:25:29,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/d0ac5dc741db46c28ae8d6a80e1968ce is 50, key is test_row_0/B:col10/1732098327510/Put/seqid=0 2024-11-20T10:25:29,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742159_1335 (size=12301) 2024-11-20T10:25:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T10:25:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:29,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. as already flushing 2024-11-20T10:25:29,658 INFO [Thread-1184 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1184. 2024-11-20T10:25:29,658 INFO [Thread-1188 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1188. 2024-11-20T10:25:29,658 DEBUG [Thread-1188 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:29,658 DEBUG [Thread-1184 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:29,658 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,658 DEBUG [Thread-1184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,658 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,658 DEBUG [Thread-1184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:29,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:29,688 INFO [Thread-1190 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1190. 2024-11-20T10:25:29,688 INFO [Thread-1182 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1182. 2024-11-20T10:25:29,688 DEBUG [Thread-1182 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:29,688 DEBUG [Thread-1190 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:29,688 DEBUG [Thread-1182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,688 DEBUG [Thread-1190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,688 DEBUG [Thread-1190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,688 DEBUG [Thread-1182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:29,688 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:29,688 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:29,826 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/d0ac5dc741db46c28ae8d6a80e1968ce 2024-11-20T10:25:29,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/da62f7e4c6a4480a8855353db76cdedf is 50, key is test_row_0/C:col10/1732098327510/Put/seqid=0 2024-11-20T10:25:29,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742160_1336 (size=12301) 2024-11-20T10:25:30,239 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/da62f7e4c6a4480a8855353db76cdedf 2024-11-20T10:25:30,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/1fa2e19457ef42dba79e961625ee48ef as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/1fa2e19457ef42dba79e961625ee48ef 2024-11-20T10:25:30,247 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/1fa2e19457ef42dba79e961625ee48ef, entries=150, sequenceid=366, filesize=12.0 K 2024-11-20T10:25:30,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/d0ac5dc741db46c28ae8d6a80e1968ce as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/d0ac5dc741db46c28ae8d6a80e1968ce 2024-11-20T10:25:30,253 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/d0ac5dc741db46c28ae8d6a80e1968ce, entries=150, sequenceid=366, filesize=12.0 K 2024-11-20T10:25:30,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/da62f7e4c6a4480a8855353db76cdedf as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/da62f7e4c6a4480a8855353db76cdedf 2024-11-20T10:25:30,257 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/da62f7e4c6a4480a8855353db76cdedf, entries=150, sequenceid=366, filesize=12.0 K 2024-11-20T10:25:30,259 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=26.84 KB/27480 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 1341ms, sequenceid=366, compaction requested=false 2024-11-20T10:25:30,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 4030da3517ebc82c8a7cacbbe3bde6e8: 2024-11-20T10:25:30,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:30,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-20T10:25:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-20T10:25:30,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-20T10:25:30,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8790 sec 2024-11-20T10:25:30,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.8850 sec 2024-11-20T10:25:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T10:25:31,547 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:37,589 INFO [Thread-1186 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1186. 2024-11-20T10:25:37,589 DEBUG [Thread-1186 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:25:37,589 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:37,589 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:37,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:37,589 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T10:25:37,589 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-20T10:25:37,589 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-20T10:25:37,589 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-20T10:25:37,589 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-20T10:25:37,589 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-20T10:25:37,589 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T10:25:37,589 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4439 2024-11-20T10:25:37,589 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4416 2024-11-20T10:25:37,589 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4322 2024-11-20T10:25:37,589 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4388 2024-11-20T10:25:37,589 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4378 2024-11-20T10:25:37,589 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T10:25:37,589 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T10:25:37,589 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T10:25:37,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:37,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:37,590 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:25:37,590 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T10:25:37,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T10:25:37,594 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098337593"}]},"ts":"1732098337593"} 2024-11-20T10:25:37,595 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T10:25:37,595 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T10:25:37,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T10:25:37,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4030da3517ebc82c8a7cacbbe3bde6e8, UNASSIGN}] 2024-11-20T10:25:37,598 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4030da3517ebc82c8a7cacbbe3bde6e8, UNASSIGN 2024-11-20T10:25:37,598 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=4030da3517ebc82c8a7cacbbe3bde6e8, regionState=CLOSING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:25:37,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4030da3517ebc82c8a7cacbbe3bde6e8, UNASSIGN because future has completed 2024-11-20T10:25:37,600 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T10:25:37,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:25:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T10:25:37,752 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:37,753 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T10:25:37,753 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 4030da3517ebc82c8a7cacbbe3bde6e8, disabling compactions & flushes 2024-11-20T10:25:37,753 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:37,753 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:37,753 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. after waiting 0 ms 2024-11-20T10:25:37,753 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:37,753 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing 4030da3517ebc82c8a7cacbbe3bde6e8 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T10:25:37,760 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/4b8c69f178754bacaa50748328246f6e is 50, key is test_row_0/A:col10/1732098329686/Put/seqid=0 2024-11-20T10:25:37,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742161_1337 (size=12301) 2024-11-20T10:25:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T10:25:38,179 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/4b8c69f178754bacaa50748328246f6e 2024-11-20T10:25:38,186 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/df5eeaa6958f45208da103d6d70c84e9 is 50, key is test_row_0/B:col10/1732098329686/Put/seqid=0 2024-11-20T10:25:38,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742162_1338 (size=12301) 2024-11-20T10:25:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T10:25:38,604 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/df5eeaa6958f45208da103d6d70c84e9 2024-11-20T10:25:38,612 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/9c8204ec44d348e3b35a320cb1409666 is 50, key is test_row_0/C:col10/1732098329686/Put/seqid=0 2024-11-20T10:25:38,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742163_1339 (size=12301) 2024-11-20T10:25:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T10:25:39,016 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/9c8204ec44d348e3b35a320cb1409666 2024-11-20T10:25:39,021 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/A/4b8c69f178754bacaa50748328246f6e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/4b8c69f178754bacaa50748328246f6e 2024-11-20T10:25:39,025 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/4b8c69f178754bacaa50748328246f6e, entries=150, sequenceid=376, filesize=12.0 K 2024-11-20T10:25:39,026 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/B/df5eeaa6958f45208da103d6d70c84e9 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/df5eeaa6958f45208da103d6d70c84e9 2024-11-20T10:25:39,030 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/df5eeaa6958f45208da103d6d70c84e9, entries=150, sequenceid=376, filesize=12.0 K 2024-11-20T10:25:39,031 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/.tmp/C/9c8204ec44d348e3b35a320cb1409666 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/9c8204ec44d348e3b35a320cb1409666 2024-11-20T10:25:39,038 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/9c8204ec44d348e3b35a320cb1409666, entries=150, sequenceid=376, filesize=12.0 K 2024-11-20T10:25:39,039 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 1286ms, sequenceid=376, compaction requested=true 2024-11-20T10:25:39,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/05aa028386d6403dbada804abfdcf6f8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/73e90a0a2bc9491eaa88cca468712c73, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/3ed2571f69df4d2982a4690986fa337f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/c91d2a6724284e0abeb748636482ea4e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/7efb3639427a4a928580a2bdab4b85d0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/dd9fe2486a9c4389842d299d00ad5fec, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/d537d2c3b1b44243a03700ce8ef535a0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/60890690a10c4bcdaae82bbcc03f9815, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/dc691a7ee16f45a994af84be47f9b2d4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/9e93adcc2e0f4cdd8169ac57bfb10ba8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/e69ef1e60eb744e2a168313a03ed4850, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/0eaa35c429914684b28d0e8d05eea9cd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/69323ac3a7b143b9911edb393259c3a2, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/3b4bf050304f49db9189800743f5a058, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/17d7e3d78add477f9556af80451dd347, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/a84a80e39a8e4f2c8908a9e05e33aa3f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/01798f701dd6485e9ccf79fc9bece3cc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/018859dd5fbd4d48ba8ed4cccf7cc1d1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/4b82bd2c75d44cb795f86d6581c99e09, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/533a7426f809449790ec26ad8e963531, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/304e4e9192cf4ee0a32b158490df0614, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/80839fa25c9f4b64b95c124849cc522e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/82ff0b73b5f5452fa9a87b3eab236c58, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/b85b81f3a8cb41db8c0747fbefc8386b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/f23b93c08257427d850e43142329729f] to archive 2024-11-20T10:25:39,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:25:39,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/05aa028386d6403dbada804abfdcf6f8 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/05aa028386d6403dbada804abfdcf6f8 2024-11-20T10:25:39,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/73e90a0a2bc9491eaa88cca468712c73 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/73e90a0a2bc9491eaa88cca468712c73 2024-11-20T10:25:39,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/3ed2571f69df4d2982a4690986fa337f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/3ed2571f69df4d2982a4690986fa337f 2024-11-20T10:25:39,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/c91d2a6724284e0abeb748636482ea4e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/c91d2a6724284e0abeb748636482ea4e 2024-11-20T10:25:39,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/7efb3639427a4a928580a2bdab4b85d0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/7efb3639427a4a928580a2bdab4b85d0 2024-11-20T10:25:39,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/dd9fe2486a9c4389842d299d00ad5fec to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/dd9fe2486a9c4389842d299d00ad5fec 2024-11-20T10:25:39,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/d537d2c3b1b44243a03700ce8ef535a0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/d537d2c3b1b44243a03700ce8ef535a0 2024-11-20T10:25:39,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/60890690a10c4bcdaae82bbcc03f9815 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/60890690a10c4bcdaae82bbcc03f9815 2024-11-20T10:25:39,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/dc691a7ee16f45a994af84be47f9b2d4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/dc691a7ee16f45a994af84be47f9b2d4 2024-11-20T10:25:39,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/9e93adcc2e0f4cdd8169ac57bfb10ba8 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/9e93adcc2e0f4cdd8169ac57bfb10ba8 2024-11-20T10:25:39,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/e69ef1e60eb744e2a168313a03ed4850 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/e69ef1e60eb744e2a168313a03ed4850 2024-11-20T10:25:39,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/0eaa35c429914684b28d0e8d05eea9cd to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/0eaa35c429914684b28d0e8d05eea9cd 2024-11-20T10:25:39,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/69323ac3a7b143b9911edb393259c3a2 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/69323ac3a7b143b9911edb393259c3a2 2024-11-20T10:25:39,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/3b4bf050304f49db9189800743f5a058 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/3b4bf050304f49db9189800743f5a058 2024-11-20T10:25:39,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/17d7e3d78add477f9556af80451dd347 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/17d7e3d78add477f9556af80451dd347 2024-11-20T10:25:39,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/a84a80e39a8e4f2c8908a9e05e33aa3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/a84a80e39a8e4f2c8908a9e05e33aa3f 2024-11-20T10:25:39,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/01798f701dd6485e9ccf79fc9bece3cc to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/01798f701dd6485e9ccf79fc9bece3cc 2024-11-20T10:25:39,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/018859dd5fbd4d48ba8ed4cccf7cc1d1 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/018859dd5fbd4d48ba8ed4cccf7cc1d1 2024-11-20T10:25:39,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/4b82bd2c75d44cb795f86d6581c99e09 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/4b82bd2c75d44cb795f86d6581c99e09 2024-11-20T10:25:39,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/533a7426f809449790ec26ad8e963531 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/533a7426f809449790ec26ad8e963531 2024-11-20T10:25:39,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/304e4e9192cf4ee0a32b158490df0614 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/304e4e9192cf4ee0a32b158490df0614 2024-11-20T10:25:39,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/80839fa25c9f4b64b95c124849cc522e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/80839fa25c9f4b64b95c124849cc522e 2024-11-20T10:25:39,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/82ff0b73b5f5452fa9a87b3eab236c58 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/82ff0b73b5f5452fa9a87b3eab236c58 2024-11-20T10:25:39,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/b85b81f3a8cb41db8c0747fbefc8386b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/b85b81f3a8cb41db8c0747fbefc8386b 2024-11-20T10:25:39,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/f23b93c08257427d850e43142329729f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/f23b93c08257427d850e43142329729f 2024-11-20T10:25:39,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/c3f08d21fb8341deaa815854942bf58d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/f770b77a17e7456cb15bf70f026be31c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/3e6c0d9fb80944a7801e648d64b75374, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/dba1ed103a074dcb96a55779333ccd04, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8486705598724862b2949489ee915d75, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/2ea0a17ff36843a8aa995f952fb513f4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/eedbfa1b4d27400da6b25a605e103fee, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/302e464254e74459886892adc423af94, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/9219a71272c34146bb9f29fd86fa9edf, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0dc75f616b37489093bbb1bcec39bc9b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/bd9878d2f0444396b416003de988d47d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0bfd67564d684c53849f71d65f884171, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/2b385cd4bb1a433e9c81dc29bf540a5b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/13f836b97082403a8717cebfe25c3db4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/ad1f558499a846d88815cb062d0b7c72, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/93ed8f04d3e24623a6b0061a30df3007, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0ee6c4ac9de1448f958d163084afceb5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/edf4f0d3a81445aaa8c87c847589b592, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/d92e8f0519a5448cab01a37b09eb28b1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8157856aa10f4695833dc8ece1a3ddb0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/19da843c349c404e8af24bd9bc399717, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/c14329e6590b481cb1f5ece31116b717, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/f2ffe30ed6c14aeda390aa80e2b84ecf, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8e9ddc37eb26425c91169899fe15840b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/6052020b111d473ca2030941c0006e6d] to archive 2024-11-20T10:25:39,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:25:39,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/c3f08d21fb8341deaa815854942bf58d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/c3f08d21fb8341deaa815854942bf58d 2024-11-20T10:25:39,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/f770b77a17e7456cb15bf70f026be31c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/f770b77a17e7456cb15bf70f026be31c 2024-11-20T10:25:39,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/3e6c0d9fb80944a7801e648d64b75374 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/3e6c0d9fb80944a7801e648d64b75374 2024-11-20T10:25:39,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/dba1ed103a074dcb96a55779333ccd04 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/dba1ed103a074dcb96a55779333ccd04 2024-11-20T10:25:39,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8486705598724862b2949489ee915d75 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8486705598724862b2949489ee915d75 2024-11-20T10:25:39,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/2ea0a17ff36843a8aa995f952fb513f4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/2ea0a17ff36843a8aa995f952fb513f4 2024-11-20T10:25:39,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/eedbfa1b4d27400da6b25a605e103fee to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/eedbfa1b4d27400da6b25a605e103fee 2024-11-20T10:25:39,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/302e464254e74459886892adc423af94 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/302e464254e74459886892adc423af94 2024-11-20T10:25:39,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/9219a71272c34146bb9f29fd86fa9edf to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/9219a71272c34146bb9f29fd86fa9edf 2024-11-20T10:25:39,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0dc75f616b37489093bbb1bcec39bc9b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0dc75f616b37489093bbb1bcec39bc9b 2024-11-20T10:25:39,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/bd9878d2f0444396b416003de988d47d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/bd9878d2f0444396b416003de988d47d 2024-11-20T10:25:39,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0bfd67564d684c53849f71d65f884171 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0bfd67564d684c53849f71d65f884171 2024-11-20T10:25:39,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/2b385cd4bb1a433e9c81dc29bf540a5b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/2b385cd4bb1a433e9c81dc29bf540a5b 2024-11-20T10:25:39,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/13f836b97082403a8717cebfe25c3db4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/13f836b97082403a8717cebfe25c3db4 2024-11-20T10:25:39,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/ad1f558499a846d88815cb062d0b7c72 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/ad1f558499a846d88815cb062d0b7c72 2024-11-20T10:25:39,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/93ed8f04d3e24623a6b0061a30df3007 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/93ed8f04d3e24623a6b0061a30df3007 2024-11-20T10:25:39,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0ee6c4ac9de1448f958d163084afceb5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/0ee6c4ac9de1448f958d163084afceb5 2024-11-20T10:25:39,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/edf4f0d3a81445aaa8c87c847589b592 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/edf4f0d3a81445aaa8c87c847589b592 2024-11-20T10:25:39,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/d92e8f0519a5448cab01a37b09eb28b1 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/d92e8f0519a5448cab01a37b09eb28b1 2024-11-20T10:25:39,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8157856aa10f4695833dc8ece1a3ddb0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8157856aa10f4695833dc8ece1a3ddb0 2024-11-20T10:25:39,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/19da843c349c404e8af24bd9bc399717 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/19da843c349c404e8af24bd9bc399717 2024-11-20T10:25:39,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/c14329e6590b481cb1f5ece31116b717 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/c14329e6590b481cb1f5ece31116b717 2024-11-20T10:25:39,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/f2ffe30ed6c14aeda390aa80e2b84ecf to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/f2ffe30ed6c14aeda390aa80e2b84ecf 2024-11-20T10:25:39,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8e9ddc37eb26425c91169899fe15840b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/8e9ddc37eb26425c91169899fe15840b 2024-11-20T10:25:39,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/6052020b111d473ca2030941c0006e6d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/6052020b111d473ca2030941c0006e6d 2024-11-20T10:25:39,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b40a429142d34b838ee9bec45e88bd8b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b9d8f00e64aa4f4d94d2ecc22947dc70, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/d892aaabec1340b69eef9744785665c7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b6335e4dd2da4132935b781d417783c5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a1adc429c1f5481c9e1d770a69dc5884, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/dbe83c21045e4a26807bd6bf7514bc63, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a05e2de6f46a40888bb1ebff57ca4f76, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/4985584a4896434eb0a6799e49488cea, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2b7ef071bae84f25b342e781b1ff9924, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/59c076a8203d4efe90511a6fb9041182, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a3afd8bd973b458cbe6576f3a08ef27a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/10d283a8ff8c4a40906915167e720b24, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/8c4830c86c2f45dbb5f86b0ce2df83eb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/26c2ef1124eb4a4e9048c59522b648c3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/f086b65dfc7c487eb65529018b2d75b6, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/65ef6a899e4f4e65bd9d3ee6a4d572ba, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/467b6fa392084ab3b9e234ce98ae4cc1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/e5931e4ee95240d8922f812123d0b43e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2de6de468ae34d098a41d504bde6f3e2, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/408f794c1106466a84d9d012921e96fd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/962febe646954f29adfd2af2f317804b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a67ffbe9b6514357a8b3c89f92033348, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/04b6eed9ffb243c08a68ee3b269f8e16, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2c2648d8cb964dc3851d698bc70ab34f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/8bd25b038164432ea0296657eac7ebe6] to archive 2024-11-20T10:25:39,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:25:39,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b40a429142d34b838ee9bec45e88bd8b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b40a429142d34b838ee9bec45e88bd8b 2024-11-20T10:25:39,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b9d8f00e64aa4f4d94d2ecc22947dc70 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b9d8f00e64aa4f4d94d2ecc22947dc70 2024-11-20T10:25:39,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/d892aaabec1340b69eef9744785665c7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/d892aaabec1340b69eef9744785665c7 2024-11-20T10:25:39,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b6335e4dd2da4132935b781d417783c5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/b6335e4dd2da4132935b781d417783c5 2024-11-20T10:25:39,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a1adc429c1f5481c9e1d770a69dc5884 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a1adc429c1f5481c9e1d770a69dc5884 2024-11-20T10:25:39,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/dbe83c21045e4a26807bd6bf7514bc63 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/dbe83c21045e4a26807bd6bf7514bc63 2024-11-20T10:25:39,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a05e2de6f46a40888bb1ebff57ca4f76 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a05e2de6f46a40888bb1ebff57ca4f76 2024-11-20T10:25:39,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/4985584a4896434eb0a6799e49488cea to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/4985584a4896434eb0a6799e49488cea 2024-11-20T10:25:39,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2b7ef071bae84f25b342e781b1ff9924 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2b7ef071bae84f25b342e781b1ff9924 2024-11-20T10:25:39,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/59c076a8203d4efe90511a6fb9041182 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/59c076a8203d4efe90511a6fb9041182 2024-11-20T10:25:39,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a3afd8bd973b458cbe6576f3a08ef27a to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a3afd8bd973b458cbe6576f3a08ef27a 2024-11-20T10:25:39,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/10d283a8ff8c4a40906915167e720b24 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/10d283a8ff8c4a40906915167e720b24 2024-11-20T10:25:39,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/8c4830c86c2f45dbb5f86b0ce2df83eb to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/8c4830c86c2f45dbb5f86b0ce2df83eb 2024-11-20T10:25:39,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/26c2ef1124eb4a4e9048c59522b648c3 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/26c2ef1124eb4a4e9048c59522b648c3 2024-11-20T10:25:39,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/f086b65dfc7c487eb65529018b2d75b6 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/f086b65dfc7c487eb65529018b2d75b6 2024-11-20T10:25:39,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/65ef6a899e4f4e65bd9d3ee6a4d572ba to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/65ef6a899e4f4e65bd9d3ee6a4d572ba 2024-11-20T10:25:39,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/467b6fa392084ab3b9e234ce98ae4cc1 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/467b6fa392084ab3b9e234ce98ae4cc1 2024-11-20T10:25:39,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/e5931e4ee95240d8922f812123d0b43e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/e5931e4ee95240d8922f812123d0b43e 2024-11-20T10:25:39,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2de6de468ae34d098a41d504bde6f3e2 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2de6de468ae34d098a41d504bde6f3e2 2024-11-20T10:25:39,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/408f794c1106466a84d9d012921e96fd to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/408f794c1106466a84d9d012921e96fd 2024-11-20T10:25:39,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/962febe646954f29adfd2af2f317804b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/962febe646954f29adfd2af2f317804b 2024-11-20T10:25:39,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a67ffbe9b6514357a8b3c89f92033348 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/a67ffbe9b6514357a8b3c89f92033348 2024-11-20T10:25:39,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/04b6eed9ffb243c08a68ee3b269f8e16 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/04b6eed9ffb243c08a68ee3b269f8e16 2024-11-20T10:25:39,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2c2648d8cb964dc3851d698bc70ab34f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/2c2648d8cb964dc3851d698bc70ab34f 2024-11-20T10:25:39,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/8bd25b038164432ea0296657eac7ebe6 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/8bd25b038164432ea0296657eac7ebe6 2024-11-20T10:25:39,138 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/recovered.edits/379.seqid, newMaxSeqId=379, maxSeqId=1 2024-11-20T10:25:39,138 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. 2024-11-20T10:25:39,139 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 4030da3517ebc82c8a7cacbbe3bde6e8: Waiting for close lock at 1732098337753Running coprocessor pre-close hooks at 1732098337753Disabling compacts and flushes for region at 1732098337753Disabling writes for close at 1732098337753Obtaining lock to block concurrent updates at 1732098337753Preparing flush snapshotting stores in 4030da3517ebc82c8a7cacbbe3bde6e8 at 1732098337753Finished memstore snapshotting TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732098337753Flushing stores of TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8. at 1732098337754 (+1 ms)Flushing 4030da3517ebc82c8a7cacbbe3bde6e8/A: creating writer at 1732098337754Flushing 4030da3517ebc82c8a7cacbbe3bde6e8/A: appending metadata at 1732098337759 (+5 ms)Flushing 4030da3517ebc82c8a7cacbbe3bde6e8/A: closing flushed file at 1732098337759Flushing 4030da3517ebc82c8a7cacbbe3bde6e8/B: creating writer at 1732098338183 (+424 ms)Flushing 4030da3517ebc82c8a7cacbbe3bde6e8/B: appending metadata at 1732098338186 (+3 ms)Flushing 4030da3517ebc82c8a7cacbbe3bde6e8/B: closing flushed file at 1732098338186Flushing 4030da3517ebc82c8a7cacbbe3bde6e8/C: creating writer at 1732098338609 (+423 ms)Flushing 4030da3517ebc82c8a7cacbbe3bde6e8/C: appending metadata at 1732098338611 (+2 ms)Flushing 4030da3517ebc82c8a7cacbbe3bde6e8/C: closing flushed file at 1732098338612 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48c90b8f: reopening flushed file at 1732098339020 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4071e428: reopening flushed file at 1732098339025 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17e4df03: reopening flushed file at 1732098339030 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 4030da3517ebc82c8a7cacbbe3bde6e8 in 1286ms, sequenceid=376, compaction requested=true at 1732098339039 (+9 ms)Writing region close event to WAL at 1732098339134 (+95 ms)Running coprocessor post-close hooks at 1732098339138 (+4 ms)Closed at 1732098339138 2024-11-20T10:25:39,141 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:39,141 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=4030da3517ebc82c8a7cacbbe3bde6e8, regionState=CLOSED 2024-11-20T10:25:39,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:25:39,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-20T10:25:39,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 4030da3517ebc82c8a7cacbbe3bde6e8, server=8c533afd331c,35833,1732098238678 in 1.5430 sec 2024-11-20T10:25:39,146 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-20T10:25:39,146 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4030da3517ebc82c8a7cacbbe3bde6e8, UNASSIGN in 1.5470 sec 2024-11-20T10:25:39,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-20T10:25:39,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5510 sec 2024-11-20T10:25:39,149 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098339149"}]},"ts":"1732098339149"} 2024-11-20T10:25:39,151 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T10:25:39,151 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T10:25:39,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5620 sec 2024-11-20T10:25:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T10:25:39,737 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:39,737 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T10:25:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:39,739 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T10:25:39,739 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:39,742 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:39,743 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/recovered.edits] 2024-11-20T10:25:39,746 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/1fa2e19457ef42dba79e961625ee48ef to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/1fa2e19457ef42dba79e961625ee48ef 2024-11-20T10:25:39,747 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/4b8c69f178754bacaa50748328246f6e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/4b8c69f178754bacaa50748328246f6e 2024-11-20T10:25:39,748 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/8099704e1f0c4dbca96f53b5acdcab1a to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/A/8099704e1f0c4dbca96f53b5acdcab1a 2024-11-20T10:25:39,750 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/5dec1da6929b4f80b8364d3dfdc17287 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/5dec1da6929b4f80b8364d3dfdc17287 2024-11-20T10:25:39,751 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/d0ac5dc741db46c28ae8d6a80e1968ce to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/d0ac5dc741db46c28ae8d6a80e1968ce 2024-11-20T10:25:39,752 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/df5eeaa6958f45208da103d6d70c84e9 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/B/df5eeaa6958f45208da103d6d70c84e9 2024-11-20T10:25:39,754 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/7ccd064387174497b29b85df492caca0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/7ccd064387174497b29b85df492caca0 2024-11-20T10:25:39,755 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/9c8204ec44d348e3b35a320cb1409666 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/9c8204ec44d348e3b35a320cb1409666 2024-11-20T10:25:39,756 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/da62f7e4c6a4480a8855353db76cdedf to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/C/da62f7e4c6a4480a8855353db76cdedf 2024-11-20T10:25:39,758 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/recovered.edits/379.seqid to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8/recovered.edits/379.seqid 2024-11-20T10:25:39,758 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/4030da3517ebc82c8a7cacbbe3bde6e8 2024-11-20T10:25:39,758 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T10:25:39,760 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:39,762 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T10:25:39,764 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T10:25:39,765 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:39,765 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T10:25:39,766 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732098339765"}]},"ts":"9223372036854775807"} 2024-11-20T10:25:39,767 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T10:25:39,768 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4030da3517ebc82c8a7cacbbe3bde6e8, NAME => 'TestAcidGuarantees,,1732098307993.4030da3517ebc82c8a7cacbbe3bde6e8.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T10:25:39,768 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T10:25:39,768 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732098339768"}]},"ts":"9223372036854775807"} 2024-11-20T10:25:39,770 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T10:25:39,771 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:39,772 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-20T10:25:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T10:25:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T10:25:39,850 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:39,860 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239 (was 242), OpenFileDescriptor=451 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=549 (was 471) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8279 (was 8630) 2024-11-20T10:25:39,870 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=549, ProcessCount=11, AvailableMemoryMB=8279 2024-11-20T10:25:39,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T10:25:39,872 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T10:25:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:39,874 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T10:25:39,874 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:39,874 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-20T10:25:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-20T10:25:39,875 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T10:25:39,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742164_1340 (size=959) 2024-11-20T10:25:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-20T10:25:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-20T10:25:40,286 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31 2024-11-20T10:25:40,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742165_1341 (size=53) 2024-11-20T10:25:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-20T10:25:40,699 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:25:40,699 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a71704901305964843e4454ac34f114c, disabling compactions & flushes 2024-11-20T10:25:40,699 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:40,699 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:40,699 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. after waiting 0 ms 2024-11-20T10:25:40,700 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:40,700 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:40,700 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a71704901305964843e4454ac34f114c: Waiting for close lock at 1732098340699Disabling compacts and flushes for region at 1732098340699Disabling writes for close at 1732098340699Writing region close event to WAL at 1732098340700 (+1 ms)Closed at 1732098340700 2024-11-20T10:25:40,702 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T10:25:40,703 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732098340702"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732098340702"}]},"ts":"1732098340702"} 2024-11-20T10:25:40,705 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T10:25:40,707 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T10:25:40,707 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098340707"}]},"ts":"1732098340707"} 2024-11-20T10:25:40,709 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T10:25:40,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71704901305964843e4454ac34f114c, ASSIGN}] 2024-11-20T10:25:40,711 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71704901305964843e4454ac34f114c, ASSIGN 2024-11-20T10:25:40,712 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71704901305964843e4454ac34f114c, ASSIGN; state=OFFLINE, location=8c533afd331c,35833,1732098238678; forceNewPlan=false, retain=false 2024-11-20T10:25:40,863 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a71704901305964843e4454ac34f114c, regionState=OPENING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:25:40,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71704901305964843e4454ac34f114c, ASSIGN because future has completed 2024-11-20T10:25:40,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:25:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-20T10:25:41,024 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:41,025 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:25:41,025 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a71704901305964843e4454ac34f114c 2024-11-20T10:25:41,025 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:25:41,025 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for a71704901305964843e4454ac34f114c 2024-11-20T10:25:41,025 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for a71704901305964843e4454ac34f114c 2024-11-20T10:25:41,028 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a71704901305964843e4454ac34f114c 2024-11-20T10:25:41,030 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71704901305964843e4454ac34f114c columnFamilyName A 2024-11-20T10:25:41,030 DEBUG [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:41,031 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] regionserver.HStore(327): Store=a71704901305964843e4454ac34f114c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:25:41,031 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a71704901305964843e4454ac34f114c 2024-11-20T10:25:41,034 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71704901305964843e4454ac34f114c columnFamilyName B 2024-11-20T10:25:41,034 DEBUG [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:41,034 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] regionserver.HStore(327): Store=a71704901305964843e4454ac34f114c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:25:41,035 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a71704901305964843e4454ac34f114c 2024-11-20T10:25:41,037 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71704901305964843e4454ac34f114c columnFamilyName C 2024-11-20T10:25:41,037 DEBUG [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:41,038 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] regionserver.HStore(327): Store=a71704901305964843e4454ac34f114c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:25:41,038 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for a71704901305964843e4454ac34f114c 2024-11-20T10:25:41,040 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c 2024-11-20T10:25:41,041 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c 2024-11-20T10:25:41,043 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for a71704901305964843e4454ac34f114c 2024-11-20T10:25:41,043 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for a71704901305964843e4454ac34f114c 2024-11-20T10:25:41,044 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T10:25:41,046 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for a71704901305964843e4454ac34f114c 2024-11-20T10:25:41,049 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:25:41,050 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened a71704901305964843e4454ac34f114c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72996134, jitterRate=0.08772715926170349}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T10:25:41,050 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a71704901305964843e4454ac34f114c 2024-11-20T10:25:41,051 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for a71704901305964843e4454ac34f114c: Running coprocessor pre-open hook at 1732098341026Writing region info on filesystem at 1732098341026Initializing all the Stores at 1732098341027 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098341027Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098341027Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098341028 (+1 ms)Cleaning up temporary data from old regions at 1732098341043 (+15 ms)Running coprocessor post-open hooks at 1732098341050 (+7 ms)Region opened successfully at 1732098341051 (+1 ms) 2024-11-20T10:25:41,052 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., pid=91, masterSystemTime=1732098341019 2024-11-20T10:25:41,055 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:41,055 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:41,056 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a71704901305964843e4454ac34f114c, regionState=OPEN, openSeqNum=2, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:25:41,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:25:41,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-20T10:25:41,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 in 193 msec 2024-11-20T10:25:41,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-20T10:25:41,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71704901305964843e4454ac34f114c, ASSIGN in 352 msec 2024-11-20T10:25:41,066 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T10:25:41,066 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098341066"}]},"ts":"1732098341066"} 2024-11-20T10:25:41,068 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T10:25:41,069 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T10:25:41,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1970 sec 2024-11-20T10:25:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-20T10:25:42,028 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:42,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24132877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:42,030 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:42,031 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52762f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:42,035 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42141a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,037 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:42,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:42,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:42,041 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T10:25:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8c533afd331c,33867,1732098237476 2024-11-20T10:25:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7ecb7abe 2024-11-20T10:25:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T10:25:42,043 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T10:25:42,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T10:25:42,045 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T10:25:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T10:25:42,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742166_1342 (size=995) 2024-11-20T10:25:42,463 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-20T10:25:42,463 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-20T10:25:42,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T10:25:42,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71704901305964843e4454ac34f114c, REOPEN/MOVE}] 2024-11-20T10:25:42,472 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71704901305964843e4454ac34f114c, REOPEN/MOVE 2024-11-20T10:25:42,473 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a71704901305964843e4454ac34f114c, regionState=CLOSING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:25:42,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71704901305964843e4454ac34f114c, REOPEN/MOVE because future has completed 2024-11-20T10:25:42,474 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T10:25:42,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:25:42,626 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close a71704901305964843e4454ac34f114c 2024-11-20T10:25:42,626 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T10:25:42,626 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing a71704901305964843e4454ac34f114c, disabling compactions & flushes 2024-11-20T10:25:42,626 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:42,627 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:42,627 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. after waiting 0 ms 2024-11-20T10:25:42,627 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:42,630 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T10:25:42,630 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:42,630 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for a71704901305964843e4454ac34f114c: Waiting for close lock at 1732098342626Running coprocessor pre-close hooks at 1732098342626Disabling compacts and flushes for region at 1732098342626Disabling writes for close at 1732098342627 (+1 ms)Writing region close event to WAL at 1732098342627Running coprocessor post-close hooks at 1732098342630 (+3 ms)Closed at 1732098342630 2024-11-20T10:25:42,631 WARN [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: a71704901305964843e4454ac34f114c to self. 2024-11-20T10:25:42,632 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed a71704901305964843e4454ac34f114c 2024-11-20T10:25:42,633 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a71704901305964843e4454ac34f114c, regionState=CLOSED 2024-11-20T10:25:42,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:25:42,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-20T10:25:42,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 in 161 msec 2024-11-20T10:25:42,637 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71704901305964843e4454ac34f114c, REOPEN/MOVE; state=CLOSED, location=8c533afd331c,35833,1732098238678; forceNewPlan=false, retain=true 2024-11-20T10:25:42,788 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a71704901305964843e4454ac34f114c, regionState=OPENING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:25:42,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71704901305964843e4454ac34f114c, REOPEN/MOVE because future has completed 2024-11-20T10:25:42,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:25:42,947 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:42,947 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:25:42,947 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a71704901305964843e4454ac34f114c 2024-11-20T10:25:42,947 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:25:42,947 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for a71704901305964843e4454ac34f114c 2024-11-20T10:25:42,947 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for a71704901305964843e4454ac34f114c 2024-11-20T10:25:42,949 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a71704901305964843e4454ac34f114c 2024-11-20T10:25:42,950 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71704901305964843e4454ac34f114c columnFamilyName A 2024-11-20T10:25:42,951 DEBUG [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:42,951 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] regionserver.HStore(327): Store=a71704901305964843e4454ac34f114c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:25:42,952 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a71704901305964843e4454ac34f114c 2024-11-20T10:25:42,952 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71704901305964843e4454ac34f114c columnFamilyName B 2024-11-20T10:25:42,952 DEBUG [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:42,953 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] regionserver.HStore(327): Store=a71704901305964843e4454ac34f114c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:25:42,953 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a71704901305964843e4454ac34f114c 2024-11-20T10:25:42,953 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71704901305964843e4454ac34f114c columnFamilyName C 2024-11-20T10:25:42,953 DEBUG [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:42,954 INFO [StoreOpener-a71704901305964843e4454ac34f114c-1 {}] regionserver.HStore(327): Store=a71704901305964843e4454ac34f114c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:25:42,954 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for a71704901305964843e4454ac34f114c 2024-11-20T10:25:42,955 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c 2024-11-20T10:25:42,956 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c 2024-11-20T10:25:42,957 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for a71704901305964843e4454ac34f114c 2024-11-20T10:25:42,957 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for a71704901305964843e4454ac34f114c 2024-11-20T10:25:42,958 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T10:25:42,959 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for a71704901305964843e4454ac34f114c 2024-11-20T10:25:42,960 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened a71704901305964843e4454ac34f114c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58781498, jitterRate=-0.12408742308616638}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T10:25:42,960 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a71704901305964843e4454ac34f114c 2024-11-20T10:25:42,960 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for a71704901305964843e4454ac34f114c: Running coprocessor pre-open hook at 1732098342948Writing region info on filesystem at 1732098342948Initializing all the Stores at 1732098342948Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098342948Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098342949 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098342949Cleaning up temporary data from old regions at 1732098342957 (+8 ms)Running coprocessor post-open hooks at 1732098342960 (+3 ms)Region opened successfully at 1732098342960 2024-11-20T10:25:42,961 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., pid=96, masterSystemTime=1732098342944 2024-11-20T10:25:42,962 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:42,963 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:42,963 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a71704901305964843e4454ac34f114c, regionState=OPEN, openSeqNum=5, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:25:42,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:25:42,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-20T10:25:42,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 in 173 msec 2024-11-20T10:25:42,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-20T10:25:42,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71704901305964843e4454ac34f114c, REOPEN/MOVE in 494 msec 2024-11-20T10:25:42,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-20T10:25:42,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 499 msec 2024-11-20T10:25:42,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 925 msec 2024-11-20T10:25:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-20T10:25:42,972 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:42,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14572f5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,973 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:42,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:42,973 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38fa4532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:42,975 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e59924f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38802cb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,976 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:42,976 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:42,976 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:42,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:42,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42598c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:42,977 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0a3fc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@209ba84b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,978 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:42,978 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:42,979 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bdefbf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:42,980 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6898c770, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7495e76b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:42,981 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:42,982 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a09de85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:42,983 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e347456, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7288b4d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:42,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:42,985 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b143b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:42,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:42,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c98298b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50946179, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:42,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:42,988 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab54cf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:42,989 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@83f98d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62931224, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:42,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:42,991 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@102e9466, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:42,992 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e694354, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@400fc310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:42,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:42,993 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20e031a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:42,994 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dbb8d76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227d6625, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:42,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:42,996 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aebb657, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:42,997 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b707027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15113514, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:25:42,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:25:42,999 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:25:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:25:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:25:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@167d8af3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:25:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:25:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:25:43,000 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:25:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31c27c55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:25:43,002 DEBUG [Thread-1570 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:43,002 DEBUG [Thread-1578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:43,002 DEBUG [Thread-1561 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:43,002 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:43,002 DEBUG [Thread-1559 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:43,002 DEBUG [Thread-1567 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:43,003 DEBUG [Thread-1574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-20T10:25:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:43,004 DEBUG [Thread-1563 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:43,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:43,004 DEBUG [Thread-1559 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:43,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:43,004 DEBUG [Thread-1559 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:43,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:43,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:43,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T10:25:43,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:43,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:43,005 DEBUG [Thread-1572 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:43,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:43,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:43,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:43,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:43,006 DEBUG [Thread-1576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:43,006 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:43,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:43,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:43,007 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:43,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:43,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:43,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:43,008 DEBUG [Thread-1565 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:25:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:25:43,009 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:25:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:25:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:25:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:25:43,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:25:43,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:25:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:25:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:25:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:25:43,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:25:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:25:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:25:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a71704901305964843e4454ac34f114c 2024-11-20T10:25:43,021 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:43,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:25:43,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ba954c65622c49fe8d526215de8d29ea_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098343017/Put/seqid=0 2024-11-20T10:25:43,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742167_1343 (size=14594) 2024-11-20T10:25:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T10:25:43,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098403108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098403108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098403110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098403110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098403112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-20T10:25:43,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:43,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:43,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:43,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098403217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098403227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098403227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098403227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098403227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-20T10:25:43,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:43,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:43,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:43,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T10:25:43,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098403427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098403436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098403436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098403436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098403436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,452 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:43,455 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ba954c65622c49fe8d526215de8d29ea_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba954c65622c49fe8d526215de8d29ea_a71704901305964843e4454ac34f114c 2024-11-20T10:25:43,456 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/a0ff35577c9c4aa6904f38f7187d40d5, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:43,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/a0ff35577c9c4aa6904f38f7187d40d5 is 175, key is test_row_0/A:col10/1732098343017/Put/seqid=0 2024-11-20T10:25:43,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742168_1344 (size=39549) 2024-11-20T10:25:43,464 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/a0ff35577c9c4aa6904f38f7187d40d5 2024-11-20T10:25:43,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-20T10:25:43,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:43,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:43,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:43,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/7a7675d1dee548d88ac11651d652b592 is 50, key is test_row_0/B:col10/1732098343017/Put/seqid=0 2024-11-20T10:25:43,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742169_1345 (size=12001) 2024-11-20T10:25:43,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/7a7675d1dee548d88ac11651d652b592 2024-11-20T10:25:43,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/33c989cb9ab44ad08288fa970e5c9d61 is 50, key is test_row_0/C:col10/1732098343017/Put/seqid=0 2024-11-20T10:25:43,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742170_1346 (size=12001) 2024-11-20T10:25:43,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-20T10:25:43,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:43,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:43,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:43,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T10:25:43,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098403736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098403747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098403747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098403747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098403747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:43,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-20T10:25:43,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:43,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:43,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:43,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/33c989cb9ab44ad08288fa970e5c9d61 2024-11-20T10:25:43,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/a0ff35577c9c4aa6904f38f7187d40d5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/a0ff35577c9c4aa6904f38f7187d40d5 2024-11-20T10:25:43,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-20T10:25:43,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:43,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:43,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:43,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/a0ff35577c9c4aa6904f38f7187d40d5, entries=200, sequenceid=15, filesize=38.6 K 2024-11-20T10:25:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:43,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/7a7675d1dee548d88ac11651d652b592 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/7a7675d1dee548d88ac11651d652b592 2024-11-20T10:25:43,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/7a7675d1dee548d88ac11651d652b592, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T10:25:43,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/33c989cb9ab44ad08288fa970e5c9d61 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/33c989cb9ab44ad08288fa970e5c9d61 2024-11-20T10:25:43,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/33c989cb9ab44ad08288fa970e5c9d61, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T10:25:43,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a71704901305964843e4454ac34f114c in 914ms, sequenceid=15, compaction requested=false 2024-11-20T10:25:43,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:44,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-20T10:25:44,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:44,076 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T10:25:44,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112011d140a6ac7a44f8a3f50196a0361b69_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098343108/Put/seqid=0 2024-11-20T10:25:44,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742171_1347 (size=12154) 2024-11-20T10:25:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T10:25:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a71704901305964843e4454ac34f114c 2024-11-20T10:25:44,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:44,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098404261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098404262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098404263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098404263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098404264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098404376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098404377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098404377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098404377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098404377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:44,490 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112011d140a6ac7a44f8a3f50196a0361b69_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112011d140a6ac7a44f8a3f50196a0361b69_a71704901305964843e4454ac34f114c 2024-11-20T10:25:44,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/1e1d066ec11b428b8ea33beb68c2741f, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:44,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/1e1d066ec11b428b8ea33beb68c2741f is 175, key is test_row_0/A:col10/1732098343108/Put/seqid=0 2024-11-20T10:25:44,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742172_1348 (size=30955) 2024-11-20T10:25:44,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098404597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098404597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098404597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098404597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098404597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,665 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T10:25:44,895 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/1e1d066ec11b428b8ea33beb68c2741f 2024-11-20T10:25:44,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/211fa26055214717826dc5b9b84c27e2 is 50, key is test_row_0/B:col10/1732098343108/Put/seqid=0 2024-11-20T10:25:44,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742173_1349 (size=12001) 2024-11-20T10:25:44,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098404906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098404917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098404917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098404917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:44,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098404917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T10:25:45,305 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/211fa26055214717826dc5b9b84c27e2 2024-11-20T10:25:45,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/fd9ff08a8d1d49afb6adff9686582e0a is 50, key is test_row_0/C:col10/1732098343108/Put/seqid=0 2024-11-20T10:25:45,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742174_1350 (size=12001) 2024-11-20T10:25:45,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098405427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:45,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098405427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:45,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098405436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:45,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098405436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:45,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098405436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:45,716 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/fd9ff08a8d1d49afb6adff9686582e0a 2024-11-20T10:25:45,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/1e1d066ec11b428b8ea33beb68c2741f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/1e1d066ec11b428b8ea33beb68c2741f 2024-11-20T10:25:45,723 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/1e1d066ec11b428b8ea33beb68c2741f, entries=150, sequenceid=40, filesize=30.2 K 2024-11-20T10:25:45,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/211fa26055214717826dc5b9b84c27e2 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/211fa26055214717826dc5b9b84c27e2 2024-11-20T10:25:45,727 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/211fa26055214717826dc5b9b84c27e2, entries=150, sequenceid=40, filesize=11.7 K 2024-11-20T10:25:45,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/fd9ff08a8d1d49afb6adff9686582e0a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/fd9ff08a8d1d49afb6adff9686582e0a 2024-11-20T10:25:45,730 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/fd9ff08a8d1d49afb6adff9686582e0a, entries=150, sequenceid=40, filesize=11.7 K 2024-11-20T10:25:45,731 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a71704901305964843e4454ac34f114c in 1655ms, sequenceid=40, compaction requested=false 2024-11-20T10:25:45,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:45,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:45,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-20T10:25:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-20T10:25:45,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-20T10:25:45,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7250 sec 2024-11-20T10:25:45,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.7320 sec 2024-11-20T10:25:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a71704901305964843e4454ac34f114c 2024-11-20T10:25:46,439 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:46,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T10:25:46,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cdf1ff2be58f493fa66d19119363f212_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098346438/Put/seqid=0 2024-11-20T10:25:46,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742175_1351 (size=14594) 2024-11-20T10:25:46,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098406475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:46,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098406476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:46,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098406476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098406481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:46,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098406482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:46,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:46,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098406586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:46,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098406596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098406597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098406597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:46,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098406597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:46,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:46,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098406796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:46,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:46,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098406817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:46,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098406817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:46,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098406817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:46,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098406817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:46,851 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:46,855 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cdf1ff2be58f493fa66d19119363f212_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cdf1ff2be58f493fa66d19119363f212_a71704901305964843e4454ac34f114c 2024-11-20T10:25:46,855 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/d191b84c3dc04e14a77c28598df0121d, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:46,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/d191b84c3dc04e14a77c28598df0121d is 175, key is test_row_0/A:col10/1732098346438/Put/seqid=0 2024-11-20T10:25:46,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742176_1352 (size=39549) 2024-11-20T10:25:47,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098407116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:47,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:47,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098407137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:47,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098407137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:47,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:47,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098407137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:47,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098407137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T10:25:47,156 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:47,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-20T10:25:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T10:25:47,158 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:47,159 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:47,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:47,262 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/d191b84c3dc04e14a77c28598df0121d 2024-11-20T10:25:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T10:25:47,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/d6ca5d5f044c4697942d3f87185f907c is 50, key is test_row_0/B:col10/1732098346438/Put/seqid=0 2024-11-20T10:25:47,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742177_1353 (size=12001) 2024-11-20T10:25:47,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T10:25:47,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:47,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:47,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:47,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:47,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:47,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T10:25:47,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:47,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:47,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:47,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:47,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T10:25:47,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T10:25:47,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:47,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:47,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:47,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:47,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:47,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098407636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:47,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098407646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:47,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098407646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:47,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098407646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:47,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098407657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:47,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/d6ca5d5f044c4697942d3f87185f907c 2024-11-20T10:25:47,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/3d22560841e44595979e4de523f8a116 is 50, key is test_row_0/C:col10/1732098346438/Put/seqid=0 2024-11-20T10:25:47,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742178_1354 (size=12001) 2024-11-20T10:25:47,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T10:25:47,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:47,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:47,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:47,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:47,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T10:25:47,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T10:25:47,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:47,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:47,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:47,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:47,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:48,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T10:25:48,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:48,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:48,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:48,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:48,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:48,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:48,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/3d22560841e44595979e4de523f8a116 2024-11-20T10:25:48,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/d191b84c3dc04e14a77c28598df0121d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/d191b84c3dc04e14a77c28598df0121d 2024-11-20T10:25:48,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/d191b84c3dc04e14a77c28598df0121d, entries=200, sequenceid=52, filesize=38.6 K 2024-11-20T10:25:48,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/d6ca5d5f044c4697942d3f87185f907c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/d6ca5d5f044c4697942d3f87185f907c 2024-11-20T10:25:48,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/d6ca5d5f044c4697942d3f87185f907c, entries=150, sequenceid=52, filesize=11.7 K 2024-11-20T10:25:48,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/3d22560841e44595979e4de523f8a116 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/3d22560841e44595979e4de523f8a116 2024-11-20T10:25:48,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/3d22560841e44595979e4de523f8a116, entries=150, sequenceid=52, filesize=11.7 K 2024-11-20T10:25:48,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a71704901305964843e4454ac34f114c in 1655ms, sequenceid=52, compaction requested=true 2024-11-20T10:25:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:25:48,095 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:25:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:25:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T10:25:48,095 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:48,096 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/A is initiating minor compaction (all files) 2024-11-20T10:25:48,096 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/A in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:48,096 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/a0ff35577c9c4aa6904f38f7187d40d5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/1e1d066ec11b428b8ea33beb68c2741f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/d191b84c3dc04e14a77c28598df0121d] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=107.5 K 2024-11-20T10:25:48,096 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:48,096 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:48,096 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/a0ff35577c9c4aa6904f38f7187d40d5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/1e1d066ec11b428b8ea33beb68c2741f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/d191b84c3dc04e14a77c28598df0121d] 2024-11-20T10:25:48,096 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0ff35577c9c4aa6904f38f7187d40d5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732098343012 2024-11-20T10:25:48,097 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e1d066ec11b428b8ea33beb68c2741f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732098343107 2024-11-20T10:25:48,097 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:48,097 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting d191b84c3dc04e14a77c28598df0121d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732098344262 2024-11-20T10:25:48,097 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/B is initiating minor compaction (all files) 2024-11-20T10:25:48,097 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/B in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:48,097 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/7a7675d1dee548d88ac11651d652b592, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/211fa26055214717826dc5b9b84c27e2, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/d6ca5d5f044c4697942d3f87185f907c] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=35.2 K 2024-11-20T10:25:48,097 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a7675d1dee548d88ac11651d652b592, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732098343012 2024-11-20T10:25:48,097 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 211fa26055214717826dc5b9b84c27e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732098343107 2024-11-20T10:25:48,098 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d6ca5d5f044c4697942d3f87185f907c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732098344262 2024-11-20T10:25:48,102 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:48,103 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411205b07d2ea15e442a1b6394b1d2cf6e30c_a71704901305964843e4454ac34f114c store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:48,104 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#B#compaction#301 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:48,104 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/f59fa3889d9644b59ae59b15e70d2547 is 50, key is test_row_0/B:col10/1732098346438/Put/seqid=0 2024-11-20T10:25:48,104 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411205b07d2ea15e442a1b6394b1d2cf6e30c_a71704901305964843e4454ac34f114c, store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:48,105 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205b07d2ea15e442a1b6394b1d2cf6e30c_a71704901305964843e4454ac34f114c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:48,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742179_1355 (size=4469) 2024-11-20T10:25:48,116 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#A#compaction#300 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:48,116 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/fe80de128abd4e408296f6e3d10dc30d is 175, key is test_row_0/A:col10/1732098346438/Put/seqid=0 2024-11-20T10:25:48,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742180_1356 (size=12104) 2024-11-20T10:25:48,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742181_1357 (size=31058) 2024-11-20T10:25:48,138 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/fe80de128abd4e408296f6e3d10dc30d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/fe80de128abd4e408296f6e3d10dc30d 2024-11-20T10:25:48,143 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/A of a71704901305964843e4454ac34f114c into fe80de128abd4e408296f6e3d10dc30d(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:48,143 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:48,143 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/A, priority=13, startTime=1732098348095; duration=0sec 2024-11-20T10:25:48,143 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:48,143 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:A 2024-11-20T10:25:48,143 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:48,145 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:48,145 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/C is initiating minor compaction (all files) 2024-11-20T10:25:48,145 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/C in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:48,145 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/33c989cb9ab44ad08288fa970e5c9d61, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/fd9ff08a8d1d49afb6adff9686582e0a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/3d22560841e44595979e4de523f8a116] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=35.2 K 2024-11-20T10:25:48,145 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33c989cb9ab44ad08288fa970e5c9d61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732098343012 2024-11-20T10:25:48,145 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd9ff08a8d1d49afb6adff9686582e0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732098343107 2024-11-20T10:25:48,146 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d22560841e44595979e4de523f8a116, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732098344262 2024-11-20T10:25:48,151 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#C#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:48,151 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/0ebd2c48871849b9bb3e9000b2d2478d is 50, key is test_row_0/C:col10/1732098346438/Put/seqid=0 2024-11-20T10:25:48,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742182_1358 (size=12104) 2024-11-20T10:25:48,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T10:25:48,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:48,226 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T10:25:48,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e3ad3a086e964c19a9d2f1c631ccaa96_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098346474/Put/seqid=0 2024-11-20T10:25:48,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742183_1359 (size=12154) 2024-11-20T10:25:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T10:25:48,528 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/f59fa3889d9644b59ae59b15e70d2547 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/f59fa3889d9644b59ae59b15e70d2547 2024-11-20T10:25:48,532 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/B of a71704901305964843e4454ac34f114c into f59fa3889d9644b59ae59b15e70d2547(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:48,532 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:48,532 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/B, priority=13, startTime=1732098348095; duration=0sec 2024-11-20T10:25:48,532 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:48,532 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:B 2024-11-20T10:25:48,558 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/0ebd2c48871849b9bb3e9000b2d2478d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/0ebd2c48871849b9bb3e9000b2d2478d 2024-11-20T10:25:48,561 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/C of a71704901305964843e4454ac34f114c into 0ebd2c48871849b9bb3e9000b2d2478d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:48,561 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:48,561 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/C, priority=13, startTime=1732098348095; duration=0sec 2024-11-20T10:25:48,561 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:48,561 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:C 2024-11-20T10:25:48,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:48,637 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e3ad3a086e964c19a9d2f1c631ccaa96_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e3ad3a086e964c19a9d2f1c631ccaa96_a71704901305964843e4454ac34f114c 2024-11-20T10:25:48,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/4604d267f39d474480395e3dcdb0355f, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:48,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/4604d267f39d474480395e3dcdb0355f is 175, key is test_row_0/A:col10/1732098346474/Put/seqid=0 2024-11-20T10:25:48,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742184_1360 (size=30955) 2024-11-20T10:25:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a71704901305964843e4454ac34f114c 2024-11-20T10:25:48,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:48,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098408677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:48,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098408678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:48,684 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in a71704901305964843e4454ac34f114c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:25:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:48,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098408679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098408680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098408681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:48,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:48,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098408786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:48,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098408797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098408797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098408797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:49,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098408996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:49,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:49,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098409007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098409007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:49,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098409007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:49,042 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/4604d267f39d474480395e3dcdb0355f 2024-11-20T10:25:49,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/e9c092b9c94f426493d20fa05a094d29 is 50, key is test_row_0/B:col10/1732098346474/Put/seqid=0 2024-11-20T10:25:49,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742185_1361 (size=12001) 2024-11-20T10:25:49,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T10:25:49,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:49,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098409316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:49,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:49,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098409326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:49,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098409326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098409326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:49,456 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/e9c092b9c94f426493d20fa05a094d29 2024-11-20T10:25:49,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/c6c5af93bd644647b850ad29a8b397e9 is 50, key is test_row_0/C:col10/1732098346474/Put/seqid=0 2024-11-20T10:25:49,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742186_1362 (size=12001) 2024-11-20T10:25:49,466 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/c6c5af93bd644647b850ad29a8b397e9 2024-11-20T10:25:49,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/4604d267f39d474480395e3dcdb0355f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/4604d267f39d474480395e3dcdb0355f 2024-11-20T10:25:49,473 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/4604d267f39d474480395e3dcdb0355f, entries=150, sequenceid=77, filesize=30.2 K 2024-11-20T10:25:49,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/e9c092b9c94f426493d20fa05a094d29 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/e9c092b9c94f426493d20fa05a094d29 2024-11-20T10:25:49,476 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/e9c092b9c94f426493d20fa05a094d29, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T10:25:49,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/c6c5af93bd644647b850ad29a8b397e9 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/c6c5af93bd644647b850ad29a8b397e9 2024-11-20T10:25:49,517 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/c6c5af93bd644647b850ad29a8b397e9, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T10:25:49,518 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for a71704901305964843e4454ac34f114c in 1293ms, sequenceid=77, compaction requested=false 2024-11-20T10:25:49,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:49,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:49,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-20T10:25:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-20T10:25:49,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-20T10:25:49,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3600 sec 2024-11-20T10:25:49,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.3640 sec 2024-11-20T10:25:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a71704901305964843e4454ac34f114c 2024-11-20T10:25:49,839 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:49,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T10:25:49,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e3aff655022049f0a2a624bd7ead6de4_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098348680/Put/seqid=0 2024-11-20T10:25:49,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742187_1363 (size=14594) 2024-11-20T10:25:49,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098409891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:49,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098409893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:49,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098409904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098409905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:50,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098410007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:50,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098410016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098410016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:50,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098410016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:50,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098410217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:50,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098410226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:50,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:50,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098410226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:50,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:50,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:50,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098410236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:50,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:50,262 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:50,266 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e3aff655022049f0a2a624bd7ead6de4_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e3aff655022049f0a2a624bd7ead6de4_a71704901305964843e4454ac34f114c 2024-11-20T10:25:50,266 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/81c91d487f4348d490c9c606060577c4, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:50,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/81c91d487f4348d490c9c606060577c4 is 175, key is test_row_0/A:col10/1732098348680/Put/seqid=0 2024-11-20T10:25:50,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742188_1364 (size=39549) 2024-11-20T10:25:50,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:50,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098410526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:50,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:50,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098410536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:50,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098410537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:50,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:50,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098410547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:50,670 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/81c91d487f4348d490c9c606060577c4 2024-11-20T10:25:50,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/d5ef2b8b947046a8979368197d71cdf1 is 50, key is test_row_0/B:col10/1732098348680/Put/seqid=0 2024-11-20T10:25:50,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742189_1365 (size=12001) 2024-11-20T10:25:50,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098410706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:50,712 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in a71704901305964843e4454ac34f114c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:25:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:51,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098411036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:51,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098411047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:51,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:51,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:51,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:51,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098411057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:51,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098411066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:51,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/d5ef2b8b947046a8979368197d71cdf1 2024-11-20T10:25:51,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/c6bac75de0e74403b3148e79568b7e11 is 50, key is test_row_0/C:col10/1732098348680/Put/seqid=0 2024-11-20T10:25:51,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742190_1366 (size=12001) 2024-11-20T10:25:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T10:25:51,317 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:51,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-20T10:25:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T10:25:51,319 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:51,320 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:51,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T10:25:51,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T10:25:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:51,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:51,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/c6bac75de0e74403b3148e79568b7e11 2024-11-20T10:25:51,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/81c91d487f4348d490c9c606060577c4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/81c91d487f4348d490c9c606060577c4 2024-11-20T10:25:51,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/81c91d487f4348d490c9c606060577c4, entries=200, sequenceid=92, filesize=38.6 K 2024-11-20T10:25:51,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/d5ef2b8b947046a8979368197d71cdf1 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/d5ef2b8b947046a8979368197d71cdf1 2024-11-20T10:25:51,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/d5ef2b8b947046a8979368197d71cdf1, entries=150, sequenceid=92, filesize=11.7 K 2024-11-20T10:25:51,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/c6bac75de0e74403b3148e79568b7e11 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/c6bac75de0e74403b3148e79568b7e11 2024-11-20T10:25:51,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/c6bac75de0e74403b3148e79568b7e11, entries=150, sequenceid=92, filesize=11.7 K 2024-11-20T10:25:51,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for a71704901305964843e4454ac34f114c in 1664ms, sequenceid=92, compaction requested=true 2024-11-20T10:25:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:25:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:51,503 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:25:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:51,503 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:25:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:51,504 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:51,504 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/A is initiating minor compaction (all files) 2024-11-20T10:25:51,504 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/A in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:51,505 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/fe80de128abd4e408296f6e3d10dc30d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/4604d267f39d474480395e3dcdb0355f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/81c91d487f4348d490c9c606060577c4] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=99.2 K 2024-11-20T10:25:51,505 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:51,505 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/fe80de128abd4e408296f6e3d10dc30d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/4604d267f39d474480395e3dcdb0355f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/81c91d487f4348d490c9c606060577c4] 2024-11-20T10:25:51,505 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:51,505 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/B is initiating minor compaction (all files) 2024-11-20T10:25:51,505 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/B in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:51,505 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/f59fa3889d9644b59ae59b15e70d2547, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/e9c092b9c94f426493d20fa05a094d29, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/d5ef2b8b947046a8979368197d71cdf1] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=35.3 K 2024-11-20T10:25:51,505 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe80de128abd4e408296f6e3d10dc30d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732098344262 2024-11-20T10:25:51,505 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f59fa3889d9644b59ae59b15e70d2547, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732098344262 2024-11-20T10:25:51,505 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4604d267f39d474480395e3dcdb0355f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732098346474 2024-11-20T10:25:51,506 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting e9c092b9c94f426493d20fa05a094d29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732098346474 2024-11-20T10:25:51,506 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81c91d487f4348d490c9c606060577c4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732098348678 2024-11-20T10:25:51,507 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d5ef2b8b947046a8979368197d71cdf1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732098348679 2024-11-20T10:25:51,513 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:51,514 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#B#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:51,516 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/339b97edc5a34bcd8ae953d7b508ad86 is 50, key is test_row_0/B:col10/1732098348680/Put/seqid=0 2024-11-20T10:25:51,521 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411206ed50f29deb2498d9a34b3d74de86836_a71704901305964843e4454ac34f114c store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:51,522 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411206ed50f29deb2498d9a34b3d74de86836_a71704901305964843e4454ac34f114c, store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:51,523 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206ed50f29deb2498d9a34b3d74de86836_a71704901305964843e4454ac34f114c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:51,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742191_1367 (size=12207) 2024-11-20T10:25:51,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742192_1368 (size=4469) 2024-11-20T10:25:51,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T10:25:51,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:51,625 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T10:25:51,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120eed6b923e96847d7bdf201d8c6d0c379_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098349889/Put/seqid=0 2024-11-20T10:25:51,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742193_1369 (size=12154) 2024-11-20T10:25:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T10:25:51,927 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#A#compaction#310 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:51,927 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/c3031737e7a84674b28a46abae5a7ddd is 175, key is test_row_0/A:col10/1732098348680/Put/seqid=0 2024-11-20T10:25:51,929 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/339b97edc5a34bcd8ae953d7b508ad86 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/339b97edc5a34bcd8ae953d7b508ad86 2024-11-20T10:25:51,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742194_1370 (size=31161) 2024-11-20T10:25:51,934 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/B of a71704901305964843e4454ac34f114c into 339b97edc5a34bcd8ae953d7b508ad86(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:51,934 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:51,934 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/B, priority=13, startTime=1732098351503; duration=0sec 2024-11-20T10:25:51,935 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:51,935 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:B 2024-11-20T10:25:51,935 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:51,935 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:51,935 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/C is initiating minor compaction (all files) 2024-11-20T10:25:51,935 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/C in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:51,936 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/0ebd2c48871849b9bb3e9000b2d2478d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/c6c5af93bd644647b850ad29a8b397e9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/c6bac75de0e74403b3148e79568b7e11] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=35.3 K 2024-11-20T10:25:51,936 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ebd2c48871849b9bb3e9000b2d2478d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732098344262 2024-11-20T10:25:51,937 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c6c5af93bd644647b850ad29a8b397e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732098346474 2024-11-20T10:25:51,937 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c6bac75de0e74403b3148e79568b7e11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732098348679 2024-11-20T10:25:51,940 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/c3031737e7a84674b28a46abae5a7ddd as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/c3031737e7a84674b28a46abae5a7ddd 2024-11-20T10:25:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T10:25:51,949 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#C#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:51,950 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/A of a71704901305964843e4454ac34f114c into c3031737e7a84674b28a46abae5a7ddd(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:51,950 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:51,950 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/A, priority=13, startTime=1732098351503; duration=0sec 2024-11-20T10:25:51,950 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:51,950 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:A 2024-11-20T10:25:51,950 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/16a31022355942b39254ece706dbdb83 is 50, key is test_row_0/C:col10/1732098348680/Put/seqid=0 2024-11-20T10:25:51,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742195_1371 (size=12207) 2024-11-20T10:25:52,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:52,038 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120eed6b923e96847d7bdf201d8c6d0c379_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120eed6b923e96847d7bdf201d8c6d0c379_a71704901305964843e4454ac34f114c 2024-11-20T10:25:52,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/bfabfccad0a44b8c8e0a0eca9321b845, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:52,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/bfabfccad0a44b8c8e0a0eca9321b845 is 175, key is test_row_0/A:col10/1732098349889/Put/seqid=0 2024-11-20T10:25:52,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742196_1372 (size=30955) 2024-11-20T10:25:52,044 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/bfabfccad0a44b8c8e0a0eca9321b845 2024-11-20T10:25:52,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/28ad847b90ab4cf5a35292e105db19e8 is 50, key is test_row_0/B:col10/1732098349889/Put/seqid=0 2024-11-20T10:25:52,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742197_1373 (size=12001) 2024-11-20T10:25:52,056 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/28ad847b90ab4cf5a35292e105db19e8 2024-11-20T10:25:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a71704901305964843e4454ac34f114c 2024-11-20T10:25:52,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:52,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/4c9ab70d8f904037a3c179b26b309ae9 is 50, key is test_row_0/C:col10/1732098349889/Put/seqid=0 2024-11-20T10:25:52,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742198_1374 (size=12001) 2024-11-20T10:25:52,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098412080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098412081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:52,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098412086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098412086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:52,090 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in a71704901305964843e4454ac34f114c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:25:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:52,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098412197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098412197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098412197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:52,358 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/16a31022355942b39254ece706dbdb83 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/16a31022355942b39254ece706dbdb83 2024-11-20T10:25:52,361 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/C of a71704901305964843e4454ac34f114c into 16a31022355942b39254ece706dbdb83(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:52,361 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:52,361 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/C, priority=13, startTime=1732098351503; duration=0sec 2024-11-20T10:25:52,362 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:52,362 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:C 2024-11-20T10:25:52,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098412407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098412407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:52,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098412407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T10:25:52,466 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/4c9ab70d8f904037a3c179b26b309ae9 2024-11-20T10:25:52,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/bfabfccad0a44b8c8e0a0eca9321b845 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/bfabfccad0a44b8c8e0a0eca9321b845 2024-11-20T10:25:52,473 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/bfabfccad0a44b8c8e0a0eca9321b845, entries=150, sequenceid=116, filesize=30.2 K 2024-11-20T10:25:52,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/28ad847b90ab4cf5a35292e105db19e8 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/28ad847b90ab4cf5a35292e105db19e8 2024-11-20T10:25:52,476 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/28ad847b90ab4cf5a35292e105db19e8, entries=150, sequenceid=116, filesize=11.7 K 2024-11-20T10:25:52,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/4c9ab70d8f904037a3c179b26b309ae9 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/4c9ab70d8f904037a3c179b26b309ae9 2024-11-20T10:25:52,480 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/4c9ab70d8f904037a3c179b26b309ae9, entries=150, sequenceid=116, filesize=11.7 K 2024-11-20T10:25:52,481 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for a71704901305964843e4454ac34f114c in 857ms, sequenceid=116, compaction requested=false 2024-11-20T10:25:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-20T10:25:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-20T10:25:52,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-20T10:25:52,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1620 sec 2024-11-20T10:25:52,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.1660 sec 2024-11-20T10:25:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a71704901305964843e4454ac34f114c 2024-11-20T10:25:52,720 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:52,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T10:25:52,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a77b76aaac1446afb3aa325a78695835_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098352719/Put/seqid=0 2024-11-20T10:25:52,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742199_1375 (size=17184) 2024-11-20T10:25:52,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098412768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098412776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:52,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098412777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:52,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098412887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098412887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:52,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098412896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:53,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098413097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:53,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098413097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:53,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098413107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:53,134 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:53,137 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a77b76aaac1446afb3aa325a78695835_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a77b76aaac1446afb3aa325a78695835_a71704901305964843e4454ac34f114c 2024-11-20T10:25:53,137 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/45486a4271274245aacb7cd58bc274c5, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:53,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/45486a4271274245aacb7cd58bc274c5 is 175, key is test_row_0/A:col10/1732098352719/Put/seqid=0 2024-11-20T10:25:53,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742200_1376 (size=48289) 2024-11-20T10:25:53,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098413416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:53,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098413416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098413416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T10:25:53,466 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:53,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-20T10:25:53,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T10:25:53,468 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:53,469 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:53,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:53,543 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/45486a4271274245aacb7cd58bc274c5 2024-11-20T10:25:53,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/405bc787f00e4589ae9f9179cd7fae81 is 50, key is test_row_0/B:col10/1732098352719/Put/seqid=0 2024-11-20T10:25:53,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742201_1377 (size=12101) 2024-11-20T10:25:53,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/405bc787f00e4589ae9f9179cd7fae81 2024-11-20T10:25:53,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/b9a2199339b5480da8c59cee340af226 is 50, key is test_row_0/C:col10/1732098352719/Put/seqid=0 2024-11-20T10:25:53,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742202_1378 (size=12101) 2024-11-20T10:25:53,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/b9a2199339b5480da8c59cee340af226 2024-11-20T10:25:53,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/45486a4271274245aacb7cd58bc274c5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/45486a4271274245aacb7cd58bc274c5 2024-11-20T10:25:53,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/45486a4271274245aacb7cd58bc274c5, entries=250, sequenceid=133, filesize=47.2 K 2024-11-20T10:25:53,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/405bc787f00e4589ae9f9179cd7fae81 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/405bc787f00e4589ae9f9179cd7fae81 2024-11-20T10:25:53,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/405bc787f00e4589ae9f9179cd7fae81, entries=150, sequenceid=133, filesize=11.8 K 2024-11-20T10:25:53,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/b9a2199339b5480da8c59cee340af226 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/b9a2199339b5480da8c59cee340af226 2024-11-20T10:25:53,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/b9a2199339b5480da8c59cee340af226, entries=150, sequenceid=133, filesize=11.8 K 2024-11-20T10:25:53,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a71704901305964843e4454ac34f114c in 853ms, sequenceid=133, compaction requested=true 2024-11-20T10:25:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:25:53,573 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:53,573 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:25:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:25:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:53,574 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:53,574 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:53,574 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/B is initiating minor compaction (all files) 2024-11-20T10:25:53,574 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/A is initiating minor compaction (all files) 2024-11-20T10:25:53,574 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/B in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:53,574 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/A in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:53,574 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/339b97edc5a34bcd8ae953d7b508ad86, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/28ad847b90ab4cf5a35292e105db19e8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/405bc787f00e4589ae9f9179cd7fae81] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=35.5 K 2024-11-20T10:25:53,574 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/c3031737e7a84674b28a46abae5a7ddd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/bfabfccad0a44b8c8e0a0eca9321b845, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/45486a4271274245aacb7cd58bc274c5] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=107.8 K 2024-11-20T10:25:53,574 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:53,574 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/c3031737e7a84674b28a46abae5a7ddd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/bfabfccad0a44b8c8e0a0eca9321b845, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/45486a4271274245aacb7cd58bc274c5] 2024-11-20T10:25:53,574 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 339b97edc5a34bcd8ae953d7b508ad86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732098348679 2024-11-20T10:25:53,574 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3031737e7a84674b28a46abae5a7ddd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732098348679 2024-11-20T10:25:53,574 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 28ad847b90ab4cf5a35292e105db19e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732098349889 2024-11-20T10:25:53,575 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfabfccad0a44b8c8e0a0eca9321b845, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732098349889 2024-11-20T10:25:53,575 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 405bc787f00e4589ae9f9179cd7fae81, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732098352070 2024-11-20T10:25:53,575 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45486a4271274245aacb7cd58bc274c5, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732098352070 2024-11-20T10:25:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T10:25:53,580 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:53,582 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#B#compaction#319 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:53,582 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/fbfb8521b1df4e359c7afe34408d0b55 is 50, key is test_row_0/B:col10/1732098352719/Put/seqid=0 2024-11-20T10:25:53,584 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411209640be013c9c41018d9742dc19d63292_a71704901305964843e4454ac34f114c store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:53,586 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411209640be013c9c41018d9742dc19d63292_a71704901305964843e4454ac34f114c, store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:53,586 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209640be013c9c41018d9742dc19d63292_a71704901305964843e4454ac34f114c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:53,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742203_1379 (size=12409) 2024-11-20T10:25:53,599 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/fbfb8521b1df4e359c7afe34408d0b55 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/fbfb8521b1df4e359c7afe34408d0b55 2024-11-20T10:25:53,604 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/B of a71704901305964843e4454ac34f114c into fbfb8521b1df4e359c7afe34408d0b55(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:53,604 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:53,604 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/B, priority=13, startTime=1732098353573; duration=0sec 2024-11-20T10:25:53,604 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:53,604 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:B 2024-11-20T10:25:53,604 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:53,605 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:53,605 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/C is initiating minor compaction (all files) 2024-11-20T10:25:53,605 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/C in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:53,605 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/16a31022355942b39254ece706dbdb83, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/4c9ab70d8f904037a3c179b26b309ae9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/b9a2199339b5480da8c59cee340af226] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=35.5 K 2024-11-20T10:25:53,606 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 16a31022355942b39254ece706dbdb83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732098348679 2024-11-20T10:25:53,606 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c9ab70d8f904037a3c179b26b309ae9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732098349889 2024-11-20T10:25:53,607 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b9a2199339b5480da8c59cee340af226, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732098352070 2024-11-20T10:25:53,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742204_1380 (size=4469) 2024-11-20T10:25:53,614 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:53,615 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/7ad9fbc78c894dc0acd035c079376660 is 50, key is test_row_0/C:col10/1732098352719/Put/seqid=0 2024-11-20T10:25:53,621 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#A#compaction#318 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:53,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T10:25:53,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:53,621 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-20T10:25:53,621 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/76fc17cf3b2645fba766eee7a63b313c is 175, key is test_row_0/A:col10/1732098352719/Put/seqid=0 2024-11-20T10:25:53,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742205_1381 (size=12409) 2024-11-20T10:25:53,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742206_1382 (size=31363) 2024-11-20T10:25:53,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a54b1e532d274c8db6c6e54532ff2b74_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098352776/Put/seqid=0 2024-11-20T10:25:53,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742207_1383 (size=12304) 2024-11-20T10:25:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T10:25:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a71704901305964843e4454ac34f114c 2024-11-20T10:25:53,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:53,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098413961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:53,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098413962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:53,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098413962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:54,029 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/7ad9fbc78c894dc0acd035c079376660 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/7ad9fbc78c894dc0acd035c079376660 2024-11-20T10:25:54,033 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/C of a71704901305964843e4454ac34f114c into 7ad9fbc78c894dc0acd035c079376660(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:54,033 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:54,033 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/C, priority=13, startTime=1732098353573; duration=0sec 2024-11-20T10:25:54,033 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:54,033 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:C 2024-11-20T10:25:54,036 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/76fc17cf3b2645fba766eee7a63b313c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/76fc17cf3b2645fba766eee7a63b313c 2024-11-20T10:25:54,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:54,040 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a54b1e532d274c8db6c6e54532ff2b74_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a54b1e532d274c8db6c6e54532ff2b74_a71704901305964843e4454ac34f114c 2024-11-20T10:25:54,041 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/A of a71704901305964843e4454ac34f114c into 76fc17cf3b2645fba766eee7a63b313c(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:54,041 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:54,041 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/A, priority=13, startTime=1732098353573; duration=0sec 2024-11-20T10:25:54,041 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:54,041 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:A 2024-11-20T10:25:54,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/8b53033fbc5e47c9bfdf7e9ac792c325, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:54,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/8b53033fbc5e47c9bfdf7e9ac792c325 is 175, key is test_row_0/A:col10/1732098352776/Put/seqid=0 2024-11-20T10:25:54,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742208_1384 (size=31105) 2024-11-20T10:25:54,049 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/8b53033fbc5e47c9bfdf7e9ac792c325 2024-11-20T10:25:54,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/4b9243a87e164d1fbf3aad315f393e77 is 50, key is test_row_0/B:col10/1732098352776/Put/seqid=0 2024-11-20T10:25:54,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742209_1385 (size=12151) 2024-11-20T10:25:54,059 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/4b9243a87e164d1fbf3aad315f393e77 2024-11-20T10:25:54,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/24b3a25dc5434cbd9337753707646c96 is 50, key is test_row_0/C:col10/1732098352776/Put/seqid=0 2024-11-20T10:25:54,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742210_1386 (size=12151) 2024-11-20T10:25:54,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098414067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:54,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098414076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:54,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:54,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098414076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:54,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T10:25:54,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098414107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:54,110 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in a71704901305964843e4454ac34f114c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:25:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:54,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098414276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:54,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:54,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098414286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:54,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098414286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:54,469 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/24b3a25dc5434cbd9337753707646c96 2024-11-20T10:25:54,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/8b53033fbc5e47c9bfdf7e9ac792c325 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/8b53033fbc5e47c9bfdf7e9ac792c325 2024-11-20T10:25:54,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098414586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:54,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:54,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098414597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:54,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098414597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T10:25:54,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732098414736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:54,737 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in a71704901305964843e4454ac34f114c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:25:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:54,747 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/8b53033fbc5e47c9bfdf7e9ac792c325, entries=150, sequenceid=156, filesize=30.4 K 2024-11-20T10:25:54,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/4b9243a87e164d1fbf3aad315f393e77 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/4b9243a87e164d1fbf3aad315f393e77 2024-11-20T10:25:54,751 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/4b9243a87e164d1fbf3aad315f393e77, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T10:25:54,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/24b3a25dc5434cbd9337753707646c96 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/24b3a25dc5434cbd9337753707646c96 2024-11-20T10:25:54,755 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/24b3a25dc5434cbd9337753707646c96, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T10:25:54,802 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for a71704901305964843e4454ac34f114c in 1181ms, sequenceid=156, compaction requested=false 2024-11-20T10:25:54,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:54,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:54,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-20T10:25:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-20T10:25:54,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-20T10:25:54,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3350 sec 2024-11-20T10:25:54,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.3390 sec 2024-11-20T10:25:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a71704901305964843e4454ac34f114c 2024-11-20T10:25:55,100 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:55,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T10:25:55,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112069bd2a4f8d4d47a39cc7b4d1db616aaa_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098353942/Put/seqid=0 2024-11-20T10:25:55,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742211_1387 (size=14794) 2024-11-20T10:25:55,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098415141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098415147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:55,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098415149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:55,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098415256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:55,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098415256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:55,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098415267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:55,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098415466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:55,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098415466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:55,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098415487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:55,513 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:55,517 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112069bd2a4f8d4d47a39cc7b4d1db616aaa_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112069bd2a4f8d4d47a39cc7b4d1db616aaa_a71704901305964843e4454ac34f114c 2024-11-20T10:25:55,517 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/f12e4c288c84432b9366298405e97598, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:55,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/f12e4c288c84432b9366298405e97598 is 175, key is test_row_0/A:col10/1732098353942/Put/seqid=0 2024-11-20T10:25:55,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742212_1388 (size=39749) 2024-11-20T10:25:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T10:25:55,617 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:55,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-20T10:25:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T10:25:55,619 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:55,620 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:55,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T10:25:55,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T10:25:55,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:55,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:55,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:55,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:55,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:55,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098415776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:55,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098415776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:55,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098415796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:55,923 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/f12e4c288c84432b9366298405e97598 2024-11-20T10:25:55,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T10:25:55,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:55,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:55,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:55,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:55,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:55,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/20eec4c8ec9844fab73dda06e3ef2f28 is 50, key is test_row_0/B:col10/1732098353942/Put/seqid=0 2024-11-20T10:25:55,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742213_1389 (size=12151) 2024-11-20T10:25:55,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/20eec4c8ec9844fab73dda06e3ef2f28 2024-11-20T10:25:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T10:25:55,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/f615537c8a3848a08c2d7b7bb1351dfc is 50, key is test_row_0/C:col10/1732098353942/Put/seqid=0 2024-11-20T10:25:55,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742214_1390 (size=12151) 2024-11-20T10:25:56,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T10:25:56,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:56,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:56,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:56,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:56,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:56,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T10:25:56,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:56,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:56,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:56,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:56,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T10:25:56,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098416286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:56,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098416287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:56,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098416307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:56,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/f615537c8a3848a08c2d7b7bb1351dfc 2024-11-20T10:25:56,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/f12e4c288c84432b9366298405e97598 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/f12e4c288c84432b9366298405e97598 2024-11-20T10:25:56,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/f12e4c288c84432b9366298405e97598, entries=200, sequenceid=174, filesize=38.8 K 2024-11-20T10:25:56,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/20eec4c8ec9844fab73dda06e3ef2f28 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/20eec4c8ec9844fab73dda06e3ef2f28 2024-11-20T10:25:56,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/20eec4c8ec9844fab73dda06e3ef2f28, entries=150, sequenceid=174, filesize=11.9 K 2024-11-20T10:25:56,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/f615537c8a3848a08c2d7b7bb1351dfc as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/f615537c8a3848a08c2d7b7bb1351dfc 2024-11-20T10:25:56,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/f615537c8a3848a08c2d7b7bb1351dfc, entries=150, sequenceid=174, filesize=11.9 K 2024-11-20T10:25:56,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for a71704901305964843e4454ac34f114c in 1257ms, sequenceid=174, compaction requested=true 2024-11-20T10:25:56,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:56,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:25:56,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:56,357 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:56,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:25:56,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:56,357 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:56,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:25:56,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:56,358 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:56,358 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:56,358 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/A is initiating minor compaction (all files) 2024-11-20T10:25:56,358 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/B is initiating minor compaction (all files) 2024-11-20T10:25:56,358 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/A in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:56,358 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/B in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:56,358 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/76fc17cf3b2645fba766eee7a63b313c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/8b53033fbc5e47c9bfdf7e9ac792c325, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/f12e4c288c84432b9366298405e97598] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=99.8 K 2024-11-20T10:25:56,358 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/fbfb8521b1df4e359c7afe34408d0b55, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/4b9243a87e164d1fbf3aad315f393e77, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/20eec4c8ec9844fab73dda06e3ef2f28] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=35.9 K 2024-11-20T10:25:56,358 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:56,358 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/76fc17cf3b2645fba766eee7a63b313c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/8b53033fbc5e47c9bfdf7e9ac792c325, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/f12e4c288c84432b9366298405e97598] 2024-11-20T10:25:56,358 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting fbfb8521b1df4e359c7afe34408d0b55, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732098352070 2024-11-20T10:25:56,358 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76fc17cf3b2645fba766eee7a63b313c, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732098352070 2024-11-20T10:25:56,359 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b9243a87e164d1fbf3aad315f393e77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732098352765 2024-11-20T10:25:56,359 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b53033fbc5e47c9bfdf7e9ac792c325, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732098352765 2024-11-20T10:25:56,359 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f12e4c288c84432b9366298405e97598, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732098353942 2024-11-20T10:25:56,359 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 20eec4c8ec9844fab73dda06e3ef2f28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732098353942 2024-11-20T10:25:56,363 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:56,365 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#B#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:56,365 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/8709767dddd9413ca7b361e1c9202f48 is 50, key is test_row_0/B:col10/1732098353942/Put/seqid=0 2024-11-20T10:25:56,365 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120873cb476092f4063a5b8b76760945f77_a71704901305964843e4454ac34f114c store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:56,367 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120873cb476092f4063a5b8b76760945f77_a71704901305964843e4454ac34f114c, store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:56,367 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120873cb476092f4063a5b8b76760945f77_a71704901305964843e4454ac34f114c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:56,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742215_1391 (size=12561) 2024-11-20T10:25:56,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742216_1392 (size=4469) 2024-11-20T10:25:56,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T10:25:56,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:56,383 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T10:25:56,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112089bd8445bb55486abe59981ff942db36_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098355138/Put/seqid=0 2024-11-20T10:25:56,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742217_1393 (size=12304) 2024-11-20T10:25:56,535 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T10:25:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T10:25:56,772 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#A#compaction#328 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:56,773 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/27707100366a4d4493dbe4bb8121ffc7 is 175, key is test_row_0/A:col10/1732098353942/Put/seqid=0 2024-11-20T10:25:56,776 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/8709767dddd9413ca7b361e1c9202f48 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/8709767dddd9413ca7b361e1c9202f48 2024-11-20T10:25:56,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742218_1394 (size=31515) 2024-11-20T10:25:56,780 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/27707100366a4d4493dbe4bb8121ffc7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/27707100366a4d4493dbe4bb8121ffc7 2024-11-20T10:25:56,783 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/A of a71704901305964843e4454ac34f114c into 27707100366a4d4493dbe4bb8121ffc7(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:56,783 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:56,783 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/A, priority=13, startTime=1732098356357; duration=0sec 2024-11-20T10:25:56,783 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:56,783 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:A 2024-11-20T10:25:56,783 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:56,784 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:56,784 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/C is initiating minor compaction (all files) 2024-11-20T10:25:56,784 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/C in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:56,784 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/7ad9fbc78c894dc0acd035c079376660, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/24b3a25dc5434cbd9337753707646c96, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/f615537c8a3848a08c2d7b7bb1351dfc] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=35.9 K 2024-11-20T10:25:56,784 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ad9fbc78c894dc0acd035c079376660, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732098352070 2024-11-20T10:25:56,785 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24b3a25dc5434cbd9337753707646c96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732098352765 2024-11-20T10:25:56,785 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f615537c8a3848a08c2d7b7bb1351dfc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732098353942 2024-11-20T10:25:56,788 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/B of a71704901305964843e4454ac34f114c into 8709767dddd9413ca7b361e1c9202f48(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:56,788 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:56,788 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/B, priority=13, startTime=1732098356357; duration=0sec 2024-11-20T10:25:56,788 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:56,788 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:B 2024-11-20T10:25:56,791 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#C#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:56,791 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/f5758ef3ad984d15af8706776bd11c24 is 50, key is test_row_0/C:col10/1732098353942/Put/seqid=0 2024-11-20T10:25:56,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:56,796 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112089bd8445bb55486abe59981ff942db36_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112089bd8445bb55486abe59981ff942db36_a71704901305964843e4454ac34f114c 2024-11-20T10:25:56,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/255e6a3af4114307b8d0c06945bd581e, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:56,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/255e6a3af4114307b8d0c06945bd581e is 175, key is test_row_0/A:col10/1732098355138/Put/seqid=0 2024-11-20T10:25:56,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742219_1395 (size=12561) 2024-11-20T10:25:56,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742220_1396 (size=31105) 2024-11-20T10:25:57,211 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/f5758ef3ad984d15af8706776bd11c24 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/f5758ef3ad984d15af8706776bd11c24 2024-11-20T10:25:57,212 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/255e6a3af4114307b8d0c06945bd581e 2024-11-20T10:25:57,215 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/C of a71704901305964843e4454ac34f114c into f5758ef3ad984d15af8706776bd11c24(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:57,215 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:57,215 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/C, priority=13, startTime=1732098356357; duration=0sec 2024-11-20T10:25:57,215 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:57,215 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:C 2024-11-20T10:25:57,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/238760aff60842c09650b95f8c0c5805 is 50, key is test_row_0/B:col10/1732098355138/Put/seqid=0 2024-11-20T10:25:57,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742221_1397 (size=12151) 2024-11-20T10:25:57,227 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/238760aff60842c09650b95f8c0c5805 2024-11-20T10:25:57,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/6c5d0bb29ca24a7e912954939e3701cc is 50, key is test_row_0/C:col10/1732098355138/Put/seqid=0 2024-11-20T10:25:57,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742222_1398 (size=12151) 2024-11-20T10:25:57,247 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/6c5d0bb29ca24a7e912954939e3701cc 2024-11-20T10:25:57,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/255e6a3af4114307b8d0c06945bd581e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/255e6a3af4114307b8d0c06945bd581e 2024-11-20T10:25:57,255 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/255e6a3af4114307b8d0c06945bd581e, entries=150, sequenceid=194, filesize=30.4 K 2024-11-20T10:25:57,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/238760aff60842c09650b95f8c0c5805 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/238760aff60842c09650b95f8c0c5805 2024-11-20T10:25:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,260 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/238760aff60842c09650b95f8c0c5805, entries=150, sequenceid=194, filesize=11.9 K 2024-11-20T10:25:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/6c5d0bb29ca24a7e912954939e3701cc as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/6c5d0bb29ca24a7e912954939e3701cc 2024-11-20T10:25:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,264 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/6c5d0bb29ca24a7e912954939e3701cc, entries=150, sequenceid=194, filesize=11.9 K 2024-11-20T10:25:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,265 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for a71704901305964843e4454ac34f114c in 882ms, sequenceid=194, compaction requested=false 2024-11-20T10:25:57,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:57,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:57,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-20T10:25:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-20T10:25:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-20T10:25:57,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6470 sec 2024-11-20T10:25:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.6510 sec 2024-11-20T10:25:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a71704901305964843e4454ac34f114c 2024-11-20T10:25:57,408 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:25:57,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:25:57,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120881a29a9c57840d2a36b4d252ca587eb_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098357407/Put/seqid=0 2024-11-20T10:25:57,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742223_1399 (size=19774) 2024-11-20T10:25:57,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098417474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:57,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:57,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098417484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:57,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:57,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098417485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:57,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098417587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:57,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098417597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:57,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:57,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098417597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:57,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:57,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T10:25:57,777 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:25:57,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:25:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-20T10:25:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T10:25:57,779 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:25:57,779 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:25:57,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:25:57,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098417797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:57,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098417807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098417807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:57,821 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:57,824 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120881a29a9c57840d2a36b4d252ca587eb_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120881a29a9c57840d2a36b4d252ca587eb_a71704901305964843e4454ac34f114c 2024-11-20T10:25:57,825 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/d31e091155c1496c8b4f3a5fec9ddc82, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:57,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/d31e091155c1496c8b4f3a5fec9ddc82 is 175, key is test_row_0/A:col10/1732098357407/Put/seqid=0 2024-11-20T10:25:57,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742224_1400 (size=57033) 2024-11-20T10:25:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T10:25:57,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T10:25:57,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:57,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:57,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:57,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:57,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T10:25:58,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:58,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:58,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:58,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T10:25:58,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098418106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:58,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:58,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:58,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:58,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098418116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:58,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098418117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:58,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:58,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732098418137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:58,144 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in a71704901305964843e4454ac34f114c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:25:58,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:58,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:58,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:58,229 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/d31e091155c1496c8b4f3a5fec9ddc82 2024-11-20T10:25:58,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/ecb5b602f21a474ab7526d614eb58953 is 50, key is test_row_0/B:col10/1732098357407/Put/seqid=0 2024-11-20T10:25:58,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T10:25:58,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:58,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:58,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:58,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742225_1401 (size=12151) 2024-11-20T10:25:58,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T10:25:58,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:58,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:58,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:58,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T10:25:58,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T10:25:58,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:58,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:58,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:58,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098418616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:58,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:58,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:58,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:58,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098418626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:58,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/ecb5b602f21a474ab7526d614eb58953 2024-11-20T10:25:58,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098418637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:58,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/6ec36e78bd8046508d057735b3f046ec is 50, key is test_row_0/C:col10/1732098357407/Put/seqid=0 2024-11-20T10:25:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:58,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742226_1402 (size=12151) 2024-11-20T10:25:58,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T10:25:58,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:58,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:58,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:58,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T10:25:58,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:58,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:58,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:58,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T10:25:59,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T10:25:59,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:59,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:59,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:59,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:59,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:59,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:25:59,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/6ec36e78bd8046508d057735b3f046ec 2024-11-20T10:25:59,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/d31e091155c1496c8b4f3a5fec9ddc82 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/d31e091155c1496c8b4f3a5fec9ddc82 2024-11-20T10:25:59,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/d31e091155c1496c8b4f3a5fec9ddc82, entries=300, sequenceid=208, filesize=55.7 K 2024-11-20T10:25:59,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/ecb5b602f21a474ab7526d614eb58953 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/ecb5b602f21a474ab7526d614eb58953 2024-11-20T10:25:59,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/ecb5b602f21a474ab7526d614eb58953, entries=150, sequenceid=208, filesize=11.9 K 2024-11-20T10:25:59,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/6ec36e78bd8046508d057735b3f046ec as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/6ec36e78bd8046508d057735b3f046ec 2024-11-20T10:25:59,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/6ec36e78bd8046508d057735b3f046ec, entries=150, sequenceid=208, filesize=11.9 K 2024-11-20T10:25:59,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for a71704901305964843e4454ac34f114c in 1654ms, sequenceid=208, compaction requested=true 2024-11-20T10:25:59,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:59,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:25:59,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:59,062 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:59,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:25:59,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:59,062 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:59,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:25:59,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:59,063 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:59,063 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:59,063 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/B is initiating minor compaction (all files) 2024-11-20T10:25:59,063 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/A is initiating minor compaction (all files) 2024-11-20T10:25:59,063 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/B in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:59,063 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/A in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:59,063 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/27707100366a4d4493dbe4bb8121ffc7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/255e6a3af4114307b8d0c06945bd581e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/d31e091155c1496c8b4f3a5fec9ddc82] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=116.8 K 2024-11-20T10:25:59,063 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/8709767dddd9413ca7b361e1c9202f48, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/238760aff60842c09650b95f8c0c5805, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/ecb5b602f21a474ab7526d614eb58953] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=36.0 K 2024-11-20T10:25:59,064 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:59,064 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/27707100366a4d4493dbe4bb8121ffc7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/255e6a3af4114307b8d0c06945bd581e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/d31e091155c1496c8b4f3a5fec9ddc82] 2024-11-20T10:25:59,064 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8709767dddd9413ca7b361e1c9202f48, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732098353942 2024-11-20T10:25:59,064 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27707100366a4d4493dbe4bb8121ffc7, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732098353942 2024-11-20T10:25:59,064 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 238760aff60842c09650b95f8c0c5805, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732098355138 2024-11-20T10:25:59,064 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 255e6a3af4114307b8d0c06945bd581e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732098355138 2024-11-20T10:25:59,064 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ecb5b602f21a474ab7526d614eb58953, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732098357397 2024-11-20T10:25:59,064 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting d31e091155c1496c8b4f3a5fec9ddc82, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732098357350 2024-11-20T10:25:59,070 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:59,070 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/6507fd4a39324621942292307eee09f7 is 50, key is test_row_0/B:col10/1732098357407/Put/seqid=0 2024-11-20T10:25:59,072 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:59,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742227_1403 (size=12663) 2024-11-20T10:25:59,085 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112073fb363efbbc41f994b36bc92f78c6c6_a71704901305964843e4454ac34f114c store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:59,087 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112073fb363efbbc41f994b36bc92f78c6c6_a71704901305964843e4454ac34f114c, store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:59,087 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112073fb363efbbc41f994b36bc92f78c6c6_a71704901305964843e4454ac34f114c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:59,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742228_1404 (size=4469) 2024-11-20T10:25:59,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T10:25:59,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:59,153 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T10:25:59,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112045607b5176c8460d9383cd4d90880ed0_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098357482/Put/seqid=0 2024-11-20T10:25:59,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742229_1405 (size=12304) 2024-11-20T10:25:59,483 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/6507fd4a39324621942292307eee09f7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/6507fd4a39324621942292307eee09f7 2024-11-20T10:25:59,486 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/B of a71704901305964843e4454ac34f114c into 6507fd4a39324621942292307eee09f7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:59,486 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:59,486 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/B, priority=13, startTime=1732098359062; duration=0sec 2024-11-20T10:25:59,487 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:25:59,487 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:B 2024-11-20T10:25:59,487 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:25:59,487 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:25:59,487 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/C is initiating minor compaction (all files) 2024-11-20T10:25:59,488 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/C in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:25:59,488 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/f5758ef3ad984d15af8706776bd11c24, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/6c5d0bb29ca24a7e912954939e3701cc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/6ec36e78bd8046508d057735b3f046ec] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=36.0 K 2024-11-20T10:25:59,488 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f5758ef3ad984d15af8706776bd11c24, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732098353942 2024-11-20T10:25:59,488 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c5d0bb29ca24a7e912954939e3701cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732098355138 2024-11-20T10:25:59,488 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ec36e78bd8046508d057735b3f046ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732098357397 2024-11-20T10:25:59,492 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#A#compaction#337 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:59,493 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/1e85a52d73e741019ab7b43eee9d0271 is 175, key is test_row_0/A:col10/1732098357407/Put/seqid=0 2024-11-20T10:25:59,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742230_1406 (size=31617) 2024-11-20T10:25:59,500 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#C#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:25:59,500 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/4d4af8d205c0419bb46462d98bd91ce4 is 50, key is test_row_0/C:col10/1732098357407/Put/seqid=0 2024-11-20T10:25:59,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742231_1407 (size=12663) 2024-11-20T10:25:59,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:25:59,568 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112045607b5176c8460d9383cd4d90880ed0_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112045607b5176c8460d9383cd4d90880ed0_a71704901305964843e4454ac34f114c 2024-11-20T10:25:59,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/c8c8e0830d7f484aa6bb8cb3cc399409, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:25:59,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/c8c8e0830d7f484aa6bb8cb3cc399409 is 175, key is test_row_0/A:col10/1732098357482/Put/seqid=0 2024-11-20T10:25:59,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742232_1408 (size=31105) 2024-11-20T10:25:59,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:25:59,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a71704901305964843e4454ac34f114c 2024-11-20T10:25:59,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:59,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098419646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:59,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:59,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:59,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:59,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:59,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098419651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:59,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:59,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:59,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:59,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:59,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098419656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:59,661 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in a71704901305964843e4454ac34f114c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:25:59,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:59,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:59,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:59,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:59,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098419757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:59,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:59,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098419766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:59,903 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/1e85a52d73e741019ab7b43eee9d0271 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/1e85a52d73e741019ab7b43eee9d0271 2024-11-20T10:25:59,908 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/A of a71704901305964843e4454ac34f114c into 1e85a52d73e741019ab7b43eee9d0271(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:59,908 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:59,908 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/A, priority=13, startTime=1732098359062; duration=0sec 2024-11-20T10:25:59,908 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:59,908 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:A 2024-11-20T10:25:59,909 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/4d4af8d205c0419bb46462d98bd91ce4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/4d4af8d205c0419bb46462d98bd91ce4 2024-11-20T10:25:59,913 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/C of a71704901305964843e4454ac34f114c into 4d4af8d205c0419bb46462d98bd91ce4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:25:59,913 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:25:59,913 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/C, priority=13, startTime=1732098359062; duration=0sec 2024-11-20T10:25:59,913 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:25:59,913 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:C 2024-11-20T10:25:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T10:25:59,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098419967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:59,974 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/c8c8e0830d7f484aa6bb8cb3cc399409 2024-11-20T10:25:59,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/0bd7fda479304e00b84f080fd5756f24 is 50, key is test_row_0/B:col10/1732098357482/Put/seqid=0 2024-11-20T10:25:59,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:25:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098419976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:25:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:25:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:25:59,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742233_1409 (size=12151) 2024-11-20T10:25:59,998 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/0bd7fda479304e00b84f080fd5756f24 2024-11-20T10:26:00,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/bdfaf18d06a6487b9d63853bd06ad3a1 is 50, key is test_row_0/C:col10/1732098357482/Put/seqid=0 2024-11-20T10:26:00,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742234_1410 (size=12151) 2024-11-20T10:26:00,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:00,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098420276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:00,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098420297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:00,418 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/bdfaf18d06a6487b9d63853bd06ad3a1 2024-11-20T10:26:00,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/c8c8e0830d7f484aa6bb8cb3cc399409 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/c8c8e0830d7f484aa6bb8cb3cc399409 2024-11-20T10:26:00,426 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/c8c8e0830d7f484aa6bb8cb3cc399409, entries=150, sequenceid=234, filesize=30.4 K 2024-11-20T10:26:00,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/0bd7fda479304e00b84f080fd5756f24 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/0bd7fda479304e00b84f080fd5756f24 2024-11-20T10:26:00,430 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/0bd7fda479304e00b84f080fd5756f24, entries=150, sequenceid=234, filesize=11.9 K 2024-11-20T10:26:00,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/bdfaf18d06a6487b9d63853bd06ad3a1 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/bdfaf18d06a6487b9d63853bd06ad3a1 2024-11-20T10:26:00,438 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/bdfaf18d06a6487b9d63853bd06ad3a1, entries=150, sequenceid=234, filesize=11.9 K 2024-11-20T10:26:00,439 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a71704901305964843e4454ac34f114c in 1286ms, sequenceid=234, compaction requested=false 2024-11-20T10:26:00,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:26:00,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:26:00,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-20T10:26:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-20T10:26:00,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-20T10:26:00,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6610 sec 2024-11-20T10:26:00,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.6660 sec 2024-11-20T10:26:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a71704901305964843e4454ac34f114c 2024-11-20T10:26:00,799 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:00,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:26:00,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dbdd76e1f492497cab6844a524d093ae_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098359650/Put/seqid=0 2024-11-20T10:26:00,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742235_1411 (size=14794) 2024-11-20T10:26:00,820 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:00,824 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dbdd76e1f492497cab6844a524d093ae_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dbdd76e1f492497cab6844a524d093ae_a71704901305964843e4454ac34f114c 2024-11-20T10:26:00,825 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/a4934dbb30ec47b2a1d2270190e24981, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:26:00,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/a4934dbb30ec47b2a1d2270190e24981 is 175, key is test_row_0/A:col10/1732098359650/Put/seqid=0 2024-11-20T10:26:00,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742236_1412 (size=39749) 2024-11-20T10:26:00,831 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/a4934dbb30ec47b2a1d2270190e24981 2024-11-20T10:26:00,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/1adc0a8fd4a949cb8e62c257e738a3e7 is 50, key is test_row_0/B:col10/1732098359650/Put/seqid=0 2024-11-20T10:26:00,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742237_1413 (size=12151) 2024-11-20T10:26:00,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098420899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:00,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098420901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:01,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098421016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:01,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098421016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:01,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098421237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:01,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:01,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098421237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:01,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:01,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/1adc0a8fd4a949cb8e62c257e738a3e7 2024-11-20T10:26:01,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/8f1591e293df42edab5b7510a3d9f03c is 50, key is test_row_0/C:col10/1732098359650/Put/seqid=0 2024-11-20T10:26:01,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742238_1414 (size=12151) 2024-11-20T10:26:01,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/8f1591e293df42edab5b7510a3d9f03c 2024-11-20T10:26:01,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/a4934dbb30ec47b2a1d2270190e24981 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/a4934dbb30ec47b2a1d2270190e24981 2024-11-20T10:26:01,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/a4934dbb30ec47b2a1d2270190e24981, entries=200, sequenceid=248, filesize=38.8 K 2024-11-20T10:26:01,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/1adc0a8fd4a949cb8e62c257e738a3e7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/1adc0a8fd4a949cb8e62c257e738a3e7 2024-11-20T10:26:01,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/1adc0a8fd4a949cb8e62c257e738a3e7, entries=150, sequenceid=248, filesize=11.9 K 2024-11-20T10:26:01,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/8f1591e293df42edab5b7510a3d9f03c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/8f1591e293df42edab5b7510a3d9f03c 2024-11-20T10:26:01,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/8f1591e293df42edab5b7510a3d9f03c, entries=150, sequenceid=248, filesize=11.9 K 2024-11-20T10:26:01,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a71704901305964843e4454ac34f114c in 521ms, sequenceid=248, compaction requested=true 2024-11-20T10:26:01,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:26:01,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:26:01,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:01,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:26:01,321 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:01,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:01,321 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:01,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:26:01,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:01,322 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:01,322 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:01,322 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/B is initiating minor compaction (all files) 2024-11-20T10:26:01,322 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/A is initiating minor compaction (all files) 2024-11-20T10:26:01,322 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/B in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:26:01,322 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/A in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:26:01,323 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/6507fd4a39324621942292307eee09f7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/0bd7fda479304e00b84f080fd5756f24, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/1adc0a8fd4a949cb8e62c257e738a3e7] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=36.1 K 2024-11-20T10:26:01,323 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/1e85a52d73e741019ab7b43eee9d0271, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/c8c8e0830d7f484aa6bb8cb3cc399409, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/a4934dbb30ec47b2a1d2270190e24981] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=100.1 K 2024-11-20T10:26:01,323 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:26:01,323 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/1e85a52d73e741019ab7b43eee9d0271, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/c8c8e0830d7f484aa6bb8cb3cc399409, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/a4934dbb30ec47b2a1d2270190e24981] 2024-11-20T10:26:01,323 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 6507fd4a39324621942292307eee09f7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732098357397 2024-11-20T10:26:01,323 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e85a52d73e741019ab7b43eee9d0271, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732098357397 2024-11-20T10:26:01,323 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bd7fda479304e00b84f080fd5756f24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732098357472 2024-11-20T10:26:01,323 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8c8e0830d7f484aa6bb8cb3cc399409, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732098357472 2024-11-20T10:26:01,323 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 1adc0a8fd4a949cb8e62c257e738a3e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732098359645 2024-11-20T10:26:01,323 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4934dbb30ec47b2a1d2270190e24981, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732098359636 2024-11-20T10:26:01,330 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:01,330 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/ab59a5d8acb24f8597c1f279de4741d3 is 50, key is test_row_0/B:col10/1732098359650/Put/seqid=0 2024-11-20T10:26:01,337 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:26:01,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742239_1415 (size=12765) 2024-11-20T10:26:01,352 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120325928b784d54c45a273950f79068e94_a71704901305964843e4454ac34f114c store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:26:01,354 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120325928b784d54c45a273950f79068e94_a71704901305964843e4454ac34f114c, store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:26:01,354 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120325928b784d54c45a273950f79068e94_a71704901305964843e4454ac34f114c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:26:01,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742240_1416 (size=4469) 2024-11-20T10:26:01,377 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#A#compaction#346 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:01,377 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/779f2c393ab94beb94720c89179d2ea6 is 175, key is test_row_0/A:col10/1732098359650/Put/seqid=0 2024-11-20T10:26:01,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742241_1417 (size=31719) 2024-11-20T10:26:01,397 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/779f2c393ab94beb94720c89179d2ea6 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/779f2c393ab94beb94720c89179d2ea6 2024-11-20T10:26:01,403 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/A of a71704901305964843e4454ac34f114c into 779f2c393ab94beb94720c89179d2ea6(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:01,403 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:26:01,403 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/A, priority=13, startTime=1732098361320; duration=0sec 2024-11-20T10:26:01,403 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:01,403 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:A 2024-11-20T10:26:01,403 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:01,405 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:01,405 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/C is initiating minor compaction (all files) 2024-11-20T10:26:01,405 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/C in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:26:01,405 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/4d4af8d205c0419bb46462d98bd91ce4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/bdfaf18d06a6487b9d63853bd06ad3a1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/8f1591e293df42edab5b7510a3d9f03c] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=36.1 K 2024-11-20T10:26:01,405 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d4af8d205c0419bb46462d98bd91ce4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732098357397 2024-11-20T10:26:01,407 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdfaf18d06a6487b9d63853bd06ad3a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732098357472 2024-11-20T10:26:01,407 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f1591e293df42edab5b7510a3d9f03c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732098359645 2024-11-20T10:26:01,413 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#C#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:01,414 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/3163dbfc08eb4fb6ac5a4d1afd87de02 is 50, key is test_row_0/C:col10/1732098359650/Put/seqid=0 2024-11-20T10:26:01,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742242_1418 (size=12765) 2024-11-20T10:26:01,564 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:01,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T10:26:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a71704901305964843e4454ac34f114c 2024-11-20T10:26:01,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120de653d39278c494983f707121534bcdc_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098360899/Put/seqid=0 2024-11-20T10:26:01,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742243_1419 (size=14994) 2024-11-20T10:26:01,582 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:01,585 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120de653d39278c494983f707121534bcdc_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120de653d39278c494983f707121534bcdc_a71704901305964843e4454ac34f114c 2024-11-20T10:26:01,586 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/e1151446db3741b2af7be774a566e601, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:26:01,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/e1151446db3741b2af7be774a566e601 is 175, key is test_row_0/A:col10/1732098360899/Put/seqid=0 2024-11-20T10:26:01,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742244_1420 (size=39949) 2024-11-20T10:26:01,592 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/e1151446db3741b2af7be774a566e601 2024-11-20T10:26:01,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098421589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:01,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098421597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:01,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/c516a7e80a804b44b5913a927a15da4f is 50, key is test_row_0/B:col10/1732098360899/Put/seqid=0 2024-11-20T10:26:01,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742245_1421 (size=12301) 2024-11-20T10:26:01,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/c516a7e80a804b44b5913a927a15da4f 2024-11-20T10:26:01,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/efa2fc4fdf45440aa8f0859b080b7472 is 50, key is test_row_0/C:col10/1732098360899/Put/seqid=0 2024-11-20T10:26:01,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742246_1422 (size=12301) 2024-11-20T10:26:01,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41586 deadline: 1732098421676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:01,684 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in a71704901305964843e4454ac34f114c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:01,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098421707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:01,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098421707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:01,744 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/ab59a5d8acb24f8597c1f279de4741d3 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/ab59a5d8acb24f8597c1f279de4741d3 2024-11-20T10:26:01,750 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/B of a71704901305964843e4454ac34f114c into ab59a5d8acb24f8597c1f279de4741d3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:01,750 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:26:01,750 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/B, priority=13, startTime=1732098361321; duration=0sec 2024-11-20T10:26:01,750 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:01,750 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:B 2024-11-20T10:26:01,822 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/3163dbfc08eb4fb6ac5a4d1afd87de02 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/3163dbfc08eb4fb6ac5a4d1afd87de02 2024-11-20T10:26:01,827 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71704901305964843e4454ac34f114c/C of a71704901305964843e4454ac34f114c into 3163dbfc08eb4fb6ac5a4d1afd87de02(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:01,827 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:26:01,827 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/C, priority=13, startTime=1732098361321; duration=0sec 2024-11-20T10:26:01,827 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:01,827 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:C 2024-11-20T10:26:01,836 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/a0ff35577c9c4aa6904f38f7187d40d5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/1e1d066ec11b428b8ea33beb68c2741f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/d191b84c3dc04e14a77c28598df0121d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/fe80de128abd4e408296f6e3d10dc30d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/4604d267f39d474480395e3dcdb0355f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/81c91d487f4348d490c9c606060577c4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/c3031737e7a84674b28a46abae5a7ddd, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/bfabfccad0a44b8c8e0a0eca9321b845, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/45486a4271274245aacb7cd58bc274c5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/76fc17cf3b2645fba766eee7a63b313c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/8b53033fbc5e47c9bfdf7e9ac792c325, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/f12e4c288c84432b9366298405e97598, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/27707100366a4d4493dbe4bb8121ffc7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/255e6a3af4114307b8d0c06945bd581e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/d31e091155c1496c8b4f3a5fec9ddc82, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/1e85a52d73e741019ab7b43eee9d0271, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/c8c8e0830d7f484aa6bb8cb3cc399409, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/a4934dbb30ec47b2a1d2270190e24981] to archive 2024-11-20T10:26:01,838 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:26:01,841 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/a0ff35577c9c4aa6904f38f7187d40d5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/a0ff35577c9c4aa6904f38f7187d40d5 2024-11-20T10:26:01,843 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/1e1d066ec11b428b8ea33beb68c2741f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/1e1d066ec11b428b8ea33beb68c2741f 2024-11-20T10:26:01,843 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/d191b84c3dc04e14a77c28598df0121d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/d191b84c3dc04e14a77c28598df0121d 2024-11-20T10:26:01,849 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/fe80de128abd4e408296f6e3d10dc30d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/fe80de128abd4e408296f6e3d10dc30d 2024-11-20T10:26:01,852 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/4604d267f39d474480395e3dcdb0355f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/4604d267f39d474480395e3dcdb0355f 2024-11-20T10:26:01,853 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/81c91d487f4348d490c9c606060577c4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/81c91d487f4348d490c9c606060577c4 2024-11-20T10:26:01,855 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/c3031737e7a84674b28a46abae5a7ddd to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/c3031737e7a84674b28a46abae5a7ddd 2024-11-20T10:26:01,856 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/bfabfccad0a44b8c8e0a0eca9321b845 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/bfabfccad0a44b8c8e0a0eca9321b845 2024-11-20T10:26:01,860 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/45486a4271274245aacb7cd58bc274c5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/45486a4271274245aacb7cd58bc274c5 2024-11-20T10:26:01,861 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/76fc17cf3b2645fba766eee7a63b313c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/76fc17cf3b2645fba766eee7a63b313c 2024-11-20T10:26:01,862 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/8b53033fbc5e47c9bfdf7e9ac792c325 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/8b53033fbc5e47c9bfdf7e9ac792c325 2024-11-20T10:26:01,863 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/f12e4c288c84432b9366298405e97598 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/f12e4c288c84432b9366298405e97598 2024-11-20T10:26:01,864 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/27707100366a4d4493dbe4bb8121ffc7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/27707100366a4d4493dbe4bb8121ffc7 2024-11-20T10:26:01,865 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/255e6a3af4114307b8d0c06945bd581e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/255e6a3af4114307b8d0c06945bd581e 2024-11-20T10:26:01,866 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/d31e091155c1496c8b4f3a5fec9ddc82 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/d31e091155c1496c8b4f3a5fec9ddc82 2024-11-20T10:26:01,868 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/1e85a52d73e741019ab7b43eee9d0271 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/1e85a52d73e741019ab7b43eee9d0271 2024-11-20T10:26:01,869 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/c8c8e0830d7f484aa6bb8cb3cc399409 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/c8c8e0830d7f484aa6bb8cb3cc399409 2024-11-20T10:26:01,870 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/a4934dbb30ec47b2a1d2270190e24981 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/a4934dbb30ec47b2a1d2270190e24981 2024-11-20T10:26:01,873 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/7a7675d1dee548d88ac11651d652b592, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/211fa26055214717826dc5b9b84c27e2, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/f59fa3889d9644b59ae59b15e70d2547, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/d6ca5d5f044c4697942d3f87185f907c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/e9c092b9c94f426493d20fa05a094d29, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/339b97edc5a34bcd8ae953d7b508ad86, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/d5ef2b8b947046a8979368197d71cdf1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/28ad847b90ab4cf5a35292e105db19e8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/fbfb8521b1df4e359c7afe34408d0b55, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/405bc787f00e4589ae9f9179cd7fae81, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/4b9243a87e164d1fbf3aad315f393e77, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/8709767dddd9413ca7b361e1c9202f48, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/20eec4c8ec9844fab73dda06e3ef2f28, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/238760aff60842c09650b95f8c0c5805, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/6507fd4a39324621942292307eee09f7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/ecb5b602f21a474ab7526d614eb58953, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/0bd7fda479304e00b84f080fd5756f24, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/1adc0a8fd4a949cb8e62c257e738a3e7] to archive 2024-11-20T10:26:01,875 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:26:01,879 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/7a7675d1dee548d88ac11651d652b592 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/7a7675d1dee548d88ac11651d652b592 2024-11-20T10:26:01,880 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/211fa26055214717826dc5b9b84c27e2 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/211fa26055214717826dc5b9b84c27e2 2024-11-20T10:26:01,882 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/f59fa3889d9644b59ae59b15e70d2547 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/f59fa3889d9644b59ae59b15e70d2547 2024-11-20T10:26:01,883 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/d6ca5d5f044c4697942d3f87185f907c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/d6ca5d5f044c4697942d3f87185f907c 2024-11-20T10:26:01,884 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/e9c092b9c94f426493d20fa05a094d29 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/e9c092b9c94f426493d20fa05a094d29 2024-11-20T10:26:01,887 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/339b97edc5a34bcd8ae953d7b508ad86 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/339b97edc5a34bcd8ae953d7b508ad86 2024-11-20T10:26:01,889 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/d5ef2b8b947046a8979368197d71cdf1 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/d5ef2b8b947046a8979368197d71cdf1 2024-11-20T10:26:01,890 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/28ad847b90ab4cf5a35292e105db19e8 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/28ad847b90ab4cf5a35292e105db19e8 2024-11-20T10:26:01,893 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/fbfb8521b1df4e359c7afe34408d0b55 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/fbfb8521b1df4e359c7afe34408d0b55 2024-11-20T10:26:01,895 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/405bc787f00e4589ae9f9179cd7fae81 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/405bc787f00e4589ae9f9179cd7fae81 2024-11-20T10:26:01,898 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/4b9243a87e164d1fbf3aad315f393e77 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/4b9243a87e164d1fbf3aad315f393e77 2024-11-20T10:26:01,899 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/8709767dddd9413ca7b361e1c9202f48 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/8709767dddd9413ca7b361e1c9202f48 2024-11-20T10:26:01,900 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/20eec4c8ec9844fab73dda06e3ef2f28 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/20eec4c8ec9844fab73dda06e3ef2f28 2024-11-20T10:26:01,901 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/238760aff60842c09650b95f8c0c5805 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/238760aff60842c09650b95f8c0c5805 2024-11-20T10:26:01,904 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/6507fd4a39324621942292307eee09f7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/6507fd4a39324621942292307eee09f7 2024-11-20T10:26:01,906 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/ecb5b602f21a474ab7526d614eb58953 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/ecb5b602f21a474ab7526d614eb58953 2024-11-20T10:26:01,908 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/0bd7fda479304e00b84f080fd5756f24 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/0bd7fda479304e00b84f080fd5756f24 2024-11-20T10:26:01,910 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/1adc0a8fd4a949cb8e62c257e738a3e7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/1adc0a8fd4a949cb8e62c257e738a3e7 2024-11-20T10:26:01,913 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/33c989cb9ab44ad08288fa970e5c9d61, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/fd9ff08a8d1d49afb6adff9686582e0a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/0ebd2c48871849b9bb3e9000b2d2478d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/3d22560841e44595979e4de523f8a116, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/c6c5af93bd644647b850ad29a8b397e9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/16a31022355942b39254ece706dbdb83, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/c6bac75de0e74403b3148e79568b7e11, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/4c9ab70d8f904037a3c179b26b309ae9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/7ad9fbc78c894dc0acd035c079376660, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/b9a2199339b5480da8c59cee340af226, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/24b3a25dc5434cbd9337753707646c96, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/f5758ef3ad984d15af8706776bd11c24, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/f615537c8a3848a08c2d7b7bb1351dfc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/6c5d0bb29ca24a7e912954939e3701cc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/4d4af8d205c0419bb46462d98bd91ce4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/6ec36e78bd8046508d057735b3f046ec, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/bdfaf18d06a6487b9d63853bd06ad3a1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/8f1591e293df42edab5b7510a3d9f03c] to archive 2024-11-20T10:26:01,917 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:26:01,929 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/33c989cb9ab44ad08288fa970e5c9d61 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/33c989cb9ab44ad08288fa970e5c9d61 2024-11-20T10:26:01,930 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/fd9ff08a8d1d49afb6adff9686582e0a to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/fd9ff08a8d1d49afb6adff9686582e0a 2024-11-20T10:26:01,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098421926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:01,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098421926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:01,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:01,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:01,933 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/0ebd2c48871849b9bb3e9000b2d2478d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/0ebd2c48871849b9bb3e9000b2d2478d 2024-11-20T10:26:01,934 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/3d22560841e44595979e4de523f8a116 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/3d22560841e44595979e4de523f8a116 2024-11-20T10:26:01,935 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/c6c5af93bd644647b850ad29a8b397e9 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/c6c5af93bd644647b850ad29a8b397e9 2024-11-20T10:26:01,936 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/16a31022355942b39254ece706dbdb83 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/16a31022355942b39254ece706dbdb83 2024-11-20T10:26:01,937 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/c6bac75de0e74403b3148e79568b7e11 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/c6bac75de0e74403b3148e79568b7e11 2024-11-20T10:26:01,937 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/4c9ab70d8f904037a3c179b26b309ae9 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/4c9ab70d8f904037a3c179b26b309ae9 2024-11-20T10:26:01,938 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/7ad9fbc78c894dc0acd035c079376660 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/7ad9fbc78c894dc0acd035c079376660 2024-11-20T10:26:01,939 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/b9a2199339b5480da8c59cee340af226 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/b9a2199339b5480da8c59cee340af226 2024-11-20T10:26:01,945 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/24b3a25dc5434cbd9337753707646c96 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/24b3a25dc5434cbd9337753707646c96 2024-11-20T10:26:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T10:26:01,947 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:01,948 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-20T10:26:01,950 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/f5758ef3ad984d15af8706776bd11c24 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/f5758ef3ad984d15af8706776bd11c24 2024-11-20T10:26:01,952 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:01,954 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:01,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T10:26:01,958 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/f615537c8a3848a08c2d7b7bb1351dfc to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/f615537c8a3848a08c2d7b7bb1351dfc 2024-11-20T10:26:01,960 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/6c5d0bb29ca24a7e912954939e3701cc to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/6c5d0bb29ca24a7e912954939e3701cc 2024-11-20T10:26:01,962 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/4d4af8d205c0419bb46462d98bd91ce4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/4d4af8d205c0419bb46462d98bd91ce4 2024-11-20T10:26:01,963 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/6ec36e78bd8046508d057735b3f046ec to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/6ec36e78bd8046508d057735b3f046ec 2024-11-20T10:26:01,964 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/bdfaf18d06a6487b9d63853bd06ad3a1 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/bdfaf18d06a6487b9d63853bd06ad3a1 2024-11-20T10:26:01,965 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/8f1591e293df42edab5b7510a3d9f03c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/8f1591e293df42edab5b7510a3d9f03c 2024-11-20T10:26:02,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/efa2fc4fdf45440aa8f0859b080b7472 2024-11-20T10:26:02,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/e1151446db3741b2af7be774a566e601 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/e1151446db3741b2af7be774a566e601 2024-11-20T10:26:02,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/e1151446db3741b2af7be774a566e601, entries=200, sequenceid=275, filesize=39.0 K 2024-11-20T10:26:02,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/c516a7e80a804b44b5913a927a15da4f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/c516a7e80a804b44b5913a927a15da4f 2024-11-20T10:26:02,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/c516a7e80a804b44b5913a927a15da4f, entries=150, sequenceid=275, filesize=12.0 K 2024-11-20T10:26:02,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/efa2fc4fdf45440aa8f0859b080b7472 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/efa2fc4fdf45440aa8f0859b080b7472 2024-11-20T10:26:02,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/efa2fc4fdf45440aa8f0859b080b7472, entries=150, sequenceid=275, filesize=12.0 K 2024-11-20T10:26:02,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a71704901305964843e4454ac34f114c in 484ms, sequenceid=275, compaction requested=false 2024-11-20T10:26:02,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:26:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T10:26:02,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T10:26:02,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:26:02,109 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-20T10:26:02,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120089e1e6c77034ea0aee0b0cbe267782b_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098361575/Put/seqid=0 2024-11-20T10:26:02,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742247_1423 (size=12454) 2024-11-20T10:26:02,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:02,127 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120089e1e6c77034ea0aee0b0cbe267782b_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120089e1e6c77034ea0aee0b0cbe267782b_a71704901305964843e4454ac34f114c 2024-11-20T10:26:02,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/cfc02dc02bb7462391022716a8152b75, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:26:02,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/cfc02dc02bb7462391022716a8152b75 is 175, key is test_row_0/A:col10/1732098361575/Put/seqid=0 2024-11-20T10:26:02,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742248_1424 (size=31255) 2024-11-20T10:26:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a71704901305964843e4454ac34f114c 2024-11-20T10:26:02,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. as already flushing 2024-11-20T10:26:02,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T10:26:02,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098422346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:02,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:02,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:02,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:02,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098422353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:02,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:02,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:02,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:02,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:02,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098422466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:02,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:02,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098422466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:02,543 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/cfc02dc02bb7462391022716a8152b75 2024-11-20T10:26:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/c121493d2eca4f84820ceeff270260a8 is 50, key is test_row_0/B:col10/1732098361575/Put/seqid=0 2024-11-20T10:26:02,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742249_1425 (size=12301) 2024-11-20T10:26:02,555 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/c121493d2eca4f84820ceeff270260a8 2024-11-20T10:26:02,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/5dab2fcef31245d0920d0faabc823796 is 50, key is test_row_0/C:col10/1732098361575/Put/seqid=0 2024-11-20T10:26:02,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742250_1426 (size=12301) 2024-11-20T10:26:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T10:26:02,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41508 deadline: 1732098422687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:02,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41546 deadline: 1732098422687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:02,967 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/5dab2fcef31245d0920d0faabc823796 2024-11-20T10:26:02,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/cfc02dc02bb7462391022716a8152b75 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/cfc02dc02bb7462391022716a8152b75 2024-11-20T10:26:02,974 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/cfc02dc02bb7462391022716a8152b75, entries=150, sequenceid=287, filesize=30.5 K 2024-11-20T10:26:02,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/c121493d2eca4f84820ceeff270260a8 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/c121493d2eca4f84820ceeff270260a8 2024-11-20T10:26:02,988 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/c121493d2eca4f84820ceeff270260a8, entries=150, sequenceid=287, filesize=12.0 K 2024-11-20T10:26:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/5dab2fcef31245d0920d0faabc823796 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/5dab2fcef31245d0920d0faabc823796 2024-11-20T10:26:02,993 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/5dab2fcef31245d0920d0faabc823796, entries=150, sequenceid=287, filesize=12.0 K 2024-11-20T10:26:02,994 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for a71704901305964843e4454ac34f114c in 885ms, sequenceid=287, compaction requested=true 2024-11-20T10:26:02,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:26:02,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:26:02,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-20T10:26:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-20T10:26:02,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-20T10:26:02,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0420 sec 2024-11-20T10:26:03,003 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:03,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-20T10:26:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on a71704901305964843e4454ac34f114c 2024-11-20T10:26:03,007 INFO [Thread-1574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1574. 2024-11-20T10:26:03,007 DEBUG [Thread-1574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:03,007 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:03,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:03,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.0530 sec 2024-11-20T10:26:03,009 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:03,010 INFO [Thread-1572 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1572. 2024-11-20T10:26:03,010 DEBUG [Thread-1572 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:03,010 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:03,010 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:03,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:03,016 INFO [Thread-1578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1578. 2024-11-20T10:26:03,016 DEBUG [Thread-1578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:03,016 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:03,016 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:03,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:03,018 INFO [Thread-1570 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1570. 2024-11-20T10:26:03,018 DEBUG [Thread-1570 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:03,018 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:03,018 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:03,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:03,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208751eadd737c46efb0f20194af6dd867_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098362343/Put/seqid=0 2024-11-20T10:26:03,019 INFO [Thread-1567 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1567. 2024-11-20T10:26:03,019 DEBUG [Thread-1567 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:03,019 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:03,019 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:03,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:03,020 INFO [Thread-1563 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1563. 2024-11-20T10:26:03,020 DEBUG [Thread-1563 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:03,020 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:03,020 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:03,020 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:03,021 INFO [Thread-1576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1576. 2024-11-20T10:26:03,021 DEBUG [Thread-1576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:03,021 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:03,021 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:03,021 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:03,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742251_1427 (size=12454) 2024-11-20T10:26:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T10:26:03,097 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:03,425 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:03,428 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208751eadd737c46efb0f20194af6dd867_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208751eadd737c46efb0f20194af6dd867_a71704901305964843e4454ac34f114c 2024-11-20T10:26:03,429 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/07b728dc890c46dc985e620d3f8153a1, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:26:03,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/07b728dc890c46dc985e620d3f8153a1 is 175, key is test_row_0/A:col10/1732098362343/Put/seqid=0 2024-11-20T10:26:03,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742252_1428 (size=31255) 2024-11-20T10:26:03,433 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/07b728dc890c46dc985e620d3f8153a1 2024-11-20T10:26:03,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/e5b1531bbadc46fc8ed2f380a8c130ed is 50, key is test_row_0/B:col10/1732098362343/Put/seqid=0 2024-11-20T10:26:03,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742253_1429 (size=12301) 2024-11-20T10:26:03,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/e5b1531bbadc46fc8ed2f380a8c130ed 2024-11-20T10:26:03,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/0acb12473b1c498987a4af80ccbf339d is 50, key is test_row_0/C:col10/1732098362343/Put/seqid=0 2024-11-20T10:26:03,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742254_1430 (size=12301) 2024-11-20T10:26:04,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/0acb12473b1c498987a4af80ccbf339d 2024-11-20T10:26:04,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/07b728dc890c46dc985e620d3f8153a1 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/07b728dc890c46dc985e620d3f8153a1 2024-11-20T10:26:04,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/07b728dc890c46dc985e620d3f8153a1, entries=150, sequenceid=314, filesize=30.5 K 2024-11-20T10:26:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/e5b1531bbadc46fc8ed2f380a8c130ed as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/e5b1531bbadc46fc8ed2f380a8c130ed 2024-11-20T10:26:04,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/e5b1531bbadc46fc8ed2f380a8c130ed, entries=150, sequenceid=314, filesize=12.0 K 2024-11-20T10:26:04,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/0acb12473b1c498987a4af80ccbf339d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/0acb12473b1c498987a4af80ccbf339d 2024-11-20T10:26:04,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/0acb12473b1c498987a4af80ccbf339d, entries=150, sequenceid=314, filesize=12.0 K 2024-11-20T10:26:04,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=13.42 KB/13740 for a71704901305964843e4454ac34f114c in 1275ms, sequenceid=314, compaction requested=true 2024-11-20T10:26:04,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:26:04,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:26:04,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:04,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:26:04,279 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:26:04,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:04,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71704901305964843e4454ac34f114c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:26:04,279 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:26:04,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:04,280 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134178 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:26:04,280 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:26:04,280 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/B is initiating minor compaction (all files) 2024-11-20T10:26:04,280 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/A is initiating minor compaction (all files) 2024-11-20T10:26:04,280 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/B in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:26:04,280 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/A in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:26:04,280 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/ab59a5d8acb24f8597c1f279de4741d3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/c516a7e80a804b44b5913a927a15da4f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/c121493d2eca4f84820ceeff270260a8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/e5b1531bbadc46fc8ed2f380a8c130ed] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=48.5 K 2024-11-20T10:26:04,280 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/779f2c393ab94beb94720c89179d2ea6, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/e1151446db3741b2af7be774a566e601, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/cfc02dc02bb7462391022716a8152b75, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/07b728dc890c46dc985e620d3f8153a1] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=131.0 K 2024-11-20T10:26:04,280 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:26:04,280 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/779f2c393ab94beb94720c89179d2ea6, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/e1151446db3741b2af7be774a566e601, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/cfc02dc02bb7462391022716a8152b75, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/07b728dc890c46dc985e620d3f8153a1] 2024-11-20T10:26:04,280 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ab59a5d8acb24f8597c1f279de4741d3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732098359645 2024-11-20T10:26:04,280 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 779f2c393ab94beb94720c89179d2ea6, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732098359645 2024-11-20T10:26:04,280 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c516a7e80a804b44b5913a927a15da4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732098360898 2024-11-20T10:26:04,280 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1151446db3741b2af7be774a566e601, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732098360889 2024-11-20T10:26:04,280 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c121493d2eca4f84820ceeff270260a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732098361575 2024-11-20T10:26:04,281 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfc02dc02bb7462391022716a8152b75, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732098361575 2024-11-20T10:26:04,281 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting e5b1531bbadc46fc8ed2f380a8c130ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732098362343 2024-11-20T10:26:04,281 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07b728dc890c46dc985e620d3f8153a1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732098362343 2024-11-20T10:26:04,288 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#B#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:04,289 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/094b2f28f8774e42bfaf0fa6674cb33e is 50, key is test_row_0/B:col10/1732098362343/Put/seqid=0 2024-11-20T10:26:04,289 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:26:04,292 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120c4398e5daf3942b8ac839c6396ad0e29_a71704901305964843e4454ac34f114c store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:26:04,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742255_1431 (size=12439) 2024-11-20T10:26:04,315 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120c4398e5daf3942b8ac839c6396ad0e29_a71704901305964843e4454ac34f114c, store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:26:04,316 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c4398e5daf3942b8ac839c6396ad0e29_a71704901305964843e4454ac34f114c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:26:04,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742256_1432 (size=4469) 2024-11-20T10:26:04,699 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/094b2f28f8774e42bfaf0fa6674cb33e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/094b2f28f8774e42bfaf0fa6674cb33e 2024-11-20T10:26:04,708 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a71704901305964843e4454ac34f114c/B of a71704901305964843e4454ac34f114c into 094b2f28f8774e42bfaf0fa6674cb33e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:04,708 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:26:04,708 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/B, priority=12, startTime=1732098364279; duration=0sec 2024-11-20T10:26:04,708 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:04,708 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:B 2024-11-20T10:26:04,708 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:26:04,709 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:26:04,709 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): a71704901305964843e4454ac34f114c/C is initiating minor compaction (all files) 2024-11-20T10:26:04,710 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71704901305964843e4454ac34f114c/C in TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:26:04,710 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/3163dbfc08eb4fb6ac5a4d1afd87de02, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/efa2fc4fdf45440aa8f0859b080b7472, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/5dab2fcef31245d0920d0faabc823796, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/0acb12473b1c498987a4af80ccbf339d] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp, totalSize=48.5 K 2024-11-20T10:26:04,710 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3163dbfc08eb4fb6ac5a4d1afd87de02, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732098359645 2024-11-20T10:26:04,710 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting efa2fc4fdf45440aa8f0859b080b7472, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732098360898 2024-11-20T10:26:04,710 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dab2fcef31245d0920d0faabc823796, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732098361575 2024-11-20T10:26:04,711 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 0acb12473b1c498987a4af80ccbf339d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732098362343 2024-11-20T10:26:04,718 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#C#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:04,718 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/c5e5393b62a24793992e8cef41197898 is 50, key is test_row_0/C:col10/1732098362343/Put/seqid=0 2024-11-20T10:26:04,720 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71704901305964843e4454ac34f114c#A#compaction#358 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:04,720 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/834cdc0a09e74c4abfa436262ea0e71a is 175, key is test_row_0/A:col10/1732098362343/Put/seqid=0 2024-11-20T10:26:04,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742257_1433 (size=12439) 2024-11-20T10:26:04,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742258_1434 (size=31393) 2024-11-20T10:26:04,788 INFO [Thread-1559 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1559. 2024-11-20T10:26:04,788 DEBUG [Thread-1559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:04,788 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:04,788 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:04,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:05,125 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/c5e5393b62a24793992e8cef41197898 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/c5e5393b62a24793992e8cef41197898 2024-11-20T10:26:05,127 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/834cdc0a09e74c4abfa436262ea0e71a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/834cdc0a09e74c4abfa436262ea0e71a 2024-11-20T10:26:05,129 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a71704901305964843e4454ac34f114c/C of a71704901305964843e4454ac34f114c into c5e5393b62a24793992e8cef41197898(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:05,129 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:26:05,129 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/C, priority=12, startTime=1732098364279; duration=0sec 2024-11-20T10:26:05,129 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:05,129 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:C 2024-11-20T10:26:05,131 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a71704901305964843e4454ac34f114c/A of a71704901305964843e4454ac34f114c into 834cdc0a09e74c4abfa436262ea0e71a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:05,131 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71704901305964843e4454ac34f114c: 2024-11-20T10:26:05,131 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., storeName=a71704901305964843e4454ac34f114c/A, priority=12, startTime=1732098364279; duration=0sec 2024-11-20T10:26:05,131 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:05,131 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71704901305964843e4454ac34f114c:A 2024-11-20T10:26:05,708 INFO [Thread-1565 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1565. 2024-11-20T10:26:05,708 DEBUG [Thread-1565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:05,708 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:05,708 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:05,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:08,178 INFO [Thread-1561 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1561. 2024-11-20T10:26:08,178 DEBUG [Thread-1561 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:08,178 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:08,178 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:08,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1466 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4397 rows 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1459 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4377 rows 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1464 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4392 rows 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1472 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4416 rows 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1464 2024-11-20T10:26:08,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4385 rows 2024-11-20T10:26:08,179 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T10:26:08,179 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T10:26:08,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:08,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:08,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:08,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T10:26:08,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:08,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T10:26:08,182 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098368182"}]},"ts":"1732098368182"} 2024-11-20T10:26:08,183 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T10:26:08,183 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T10:26:08,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T10:26:08,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71704901305964843e4454ac34f114c, UNASSIGN}] 2024-11-20T10:26:08,186 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71704901305964843e4454ac34f114c, UNASSIGN 2024-11-20T10:26:08,187 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=a71704901305964843e4454ac34f114c, regionState=CLOSING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:26:08,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71704901305964843e4454ac34f114c, UNASSIGN because future has completed 2024-11-20T10:26:08,188 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T10:26:08,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:26:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T10:26:08,340 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close a71704901305964843e4454ac34f114c 2024-11-20T10:26:08,340 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T10:26:08,340 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing a71704901305964843e4454ac34f114c, disabling compactions & flushes 2024-11-20T10:26:08,340 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:26:08,340 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:26:08,340 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. after waiting 0 ms 2024-11-20T10:26:08,340 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:26:08,340 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing a71704901305964843e4454ac34f114c 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T10:26:08,345 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203732ed43270547878573d28afb0c9473_a71704901305964843e4454ac34f114c is 50, key is test_row_0/A:col10/1732098363016/Put/seqid=0 2024-11-20T10:26:08,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742259_1435 (size=9914) 2024-11-20T10:26:08,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T10:26:08,749 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:08,757 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203732ed43270547878573d28afb0c9473_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203732ed43270547878573d28afb0c9473_a71704901305964843e4454ac34f114c 2024-11-20T10:26:08,758 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/799180ebad874db2a70dd17f0805356f, store: [table=TestAcidGuarantees family=A region=a71704901305964843e4454ac34f114c] 2024-11-20T10:26:08,759 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/799180ebad874db2a70dd17f0805356f is 175, key is test_row_0/A:col10/1732098363016/Put/seqid=0 2024-11-20T10:26:08,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742260_1436 (size=22561) 2024-11-20T10:26:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T10:26:09,164 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/799180ebad874db2a70dd17f0805356f 2024-11-20T10:26:09,177 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/372c075cb118407d921b686802f1f618 is 50, key is test_row_0/B:col10/1732098363016/Put/seqid=0 2024-11-20T10:26:09,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742261_1437 (size=9857) 2024-11-20T10:26:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T10:26:09,583 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/372c075cb118407d921b686802f1f618 2024-11-20T10:26:09,596 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/49b09488fc01429882fb8e0b5d516160 is 50, key is test_row_0/C:col10/1732098363016/Put/seqid=0 2024-11-20T10:26:09,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742262_1438 (size=9857) 2024-11-20T10:26:10,002 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/49b09488fc01429882fb8e0b5d516160 2024-11-20T10:26:10,006 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/A/799180ebad874db2a70dd17f0805356f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/799180ebad874db2a70dd17f0805356f 2024-11-20T10:26:10,009 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/799180ebad874db2a70dd17f0805356f, entries=100, sequenceid=325, filesize=22.0 K 2024-11-20T10:26:10,010 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/B/372c075cb118407d921b686802f1f618 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/372c075cb118407d921b686802f1f618 2024-11-20T10:26:10,013 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/372c075cb118407d921b686802f1f618, entries=100, sequenceid=325, filesize=9.6 K 2024-11-20T10:26:10,014 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/.tmp/C/49b09488fc01429882fb8e0b5d516160 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/49b09488fc01429882fb8e0b5d516160 2024-11-20T10:26:10,017 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/49b09488fc01429882fb8e0b5d516160, entries=100, sequenceid=325, filesize=9.6 K 2024-11-20T10:26:10,018 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a71704901305964843e4454ac34f114c in 1677ms, sequenceid=325, compaction requested=false 2024-11-20T10:26:10,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/779f2c393ab94beb94720c89179d2ea6, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/e1151446db3741b2af7be774a566e601, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/cfc02dc02bb7462391022716a8152b75, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/07b728dc890c46dc985e620d3f8153a1] to archive 2024-11-20T10:26:10,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:26:10,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/779f2c393ab94beb94720c89179d2ea6 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/779f2c393ab94beb94720c89179d2ea6 2024-11-20T10:26:10,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/e1151446db3741b2af7be774a566e601 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/e1151446db3741b2af7be774a566e601 2024-11-20T10:26:10,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/cfc02dc02bb7462391022716a8152b75 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/cfc02dc02bb7462391022716a8152b75 2024-11-20T10:26:10,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/07b728dc890c46dc985e620d3f8153a1 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/07b728dc890c46dc985e620d3f8153a1 2024-11-20T10:26:10,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/ab59a5d8acb24f8597c1f279de4741d3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/c516a7e80a804b44b5913a927a15da4f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/c121493d2eca4f84820ceeff270260a8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/e5b1531bbadc46fc8ed2f380a8c130ed] to archive 2024-11-20T10:26:10,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:26:10,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/ab59a5d8acb24f8597c1f279de4741d3 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/ab59a5d8acb24f8597c1f279de4741d3 2024-11-20T10:26:10,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/c516a7e80a804b44b5913a927a15da4f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/c516a7e80a804b44b5913a927a15da4f 2024-11-20T10:26:10,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/c121493d2eca4f84820ceeff270260a8 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/c121493d2eca4f84820ceeff270260a8 2024-11-20T10:26:10,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/e5b1531bbadc46fc8ed2f380a8c130ed to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/e5b1531bbadc46fc8ed2f380a8c130ed 2024-11-20T10:26:10,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/3163dbfc08eb4fb6ac5a4d1afd87de02, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/efa2fc4fdf45440aa8f0859b080b7472, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/5dab2fcef31245d0920d0faabc823796, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/0acb12473b1c498987a4af80ccbf339d] to archive 2024-11-20T10:26:10,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:26:10,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/3163dbfc08eb4fb6ac5a4d1afd87de02 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/3163dbfc08eb4fb6ac5a4d1afd87de02 2024-11-20T10:26:10,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/efa2fc4fdf45440aa8f0859b080b7472 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/efa2fc4fdf45440aa8f0859b080b7472 2024-11-20T10:26:10,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/5dab2fcef31245d0920d0faabc823796 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/5dab2fcef31245d0920d0faabc823796 2024-11-20T10:26:10,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/0acb12473b1c498987a4af80ccbf339d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/0acb12473b1c498987a4af80ccbf339d 2024-11-20T10:26:10,037 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/recovered.edits/328.seqid, newMaxSeqId=328, maxSeqId=4 2024-11-20T10:26:10,037 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. 2024-11-20T10:26:10,037 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for a71704901305964843e4454ac34f114c: Waiting for close lock at 1732098368340Running coprocessor pre-close hooks at 1732098368340Disabling compacts and flushes for region at 1732098368340Disabling writes for close at 1732098368340Obtaining lock to block concurrent updates at 1732098368340Preparing flush snapshotting stores in a71704901305964843e4454ac34f114c at 1732098368340Finished memstore snapshotting TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732098368341 (+1 ms)Flushing stores of TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c. at 1732098368341Flushing a71704901305964843e4454ac34f114c/A: creating writer at 1732098368341Flushing mob file a71704901305964843e4454ac34f114c/A: appending metadata at 1732098368345 (+4 ms)Flushing mob file a71704901305964843e4454ac34f114c/A: closing flushed file at 1732098368345Flushing a71704901305964843e4454ac34f114c/A: appending metadata at 1732098368758 (+413 ms)Flushing a71704901305964843e4454ac34f114c/A: closing flushed file at 1732098368759 (+1 ms)Flushing a71704901305964843e4454ac34f114c/B: creating writer at 1732098369172 (+413 ms)Flushing a71704901305964843e4454ac34f114c/B: appending metadata at 1732098369176 (+4 ms)Flushing a71704901305964843e4454ac34f114c/B: closing flushed file at 1732098369176Flushing a71704901305964843e4454ac34f114c/C: creating writer at 1732098369590 (+414 ms)Flushing a71704901305964843e4454ac34f114c/C: appending metadata at 1732098369595 (+5 ms)Flushing a71704901305964843e4454ac34f114c/C: closing flushed file at 1732098369595Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75cc1626: reopening flushed file at 1732098370005 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31ac56fb: reopening flushed file at 1732098370009 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23868e53: reopening flushed file at 1732098370013 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a71704901305964843e4454ac34f114c in 1677ms, sequenceid=325, compaction requested=false at 1732098370018 (+5 ms)Writing region close event to WAL at 1732098370034 (+16 ms)Running coprocessor post-close hooks at 1732098370037 (+3 ms)Closed at 1732098370037 2024-11-20T10:26:10,039 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,040 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=a71704901305964843e4454ac34f114c, regionState=CLOSED 2024-11-20T10:26:10,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:26:10,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-20T10:26:10,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure a71704901305964843e4454ac34f114c, server=8c533afd331c,35833,1732098238678 in 1.8530 sec 2024-11-20T10:26:10,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-20T10:26:10,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71704901305964843e4454ac34f114c, UNASSIGN in 1.8580 sec 2024-11-20T10:26:10,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-20T10:26:10,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8600 sec 2024-11-20T10:26:10,046 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098370046"}]},"ts":"1732098370046"} 2024-11-20T10:26:10,047 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T10:26:10,047 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T10:26:10,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8680 sec 2024-11-20T10:26:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T10:26:10,327 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:10,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T10:26:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:10,330 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:10,331 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T10:26:10,336 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,338 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/recovered.edits] 2024-11-20T10:26:10,347 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/799180ebad874db2a70dd17f0805356f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/799180ebad874db2a70dd17f0805356f 2024-11-20T10:26:10,350 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/834cdc0a09e74c4abfa436262ea0e71a to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/A/834cdc0a09e74c4abfa436262ea0e71a 2024-11-20T10:26:10,357 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/094b2f28f8774e42bfaf0fa6674cb33e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/094b2f28f8774e42bfaf0fa6674cb33e 2024-11-20T10:26:10,359 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/372c075cb118407d921b686802f1f618 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/B/372c075cb118407d921b686802f1f618 2024-11-20T10:26:10,373 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/49b09488fc01429882fb8e0b5d516160 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/49b09488fc01429882fb8e0b5d516160 2024-11-20T10:26:10,374 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/c5e5393b62a24793992e8cef41197898 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/C/c5e5393b62a24793992e8cef41197898 2024-11-20T10:26:10,377 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/recovered.edits/328.seqid to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c/recovered.edits/328.seqid 2024-11-20T10:26:10,379 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,379 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T10:26:10,380 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T10:26:10,381 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T10:26:10,390 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120089e1e6c77034ea0aee0b0cbe267782b_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120089e1e6c77034ea0aee0b0cbe267782b_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,391 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112011d140a6ac7a44f8a3f50196a0361b69_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112011d140a6ac7a44f8a3f50196a0361b69_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,395 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203732ed43270547878573d28afb0c9473_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203732ed43270547878573d28afb0c9473_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,397 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112045607b5176c8460d9383cd4d90880ed0_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112045607b5176c8460d9383cd4d90880ed0_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,398 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112069bd2a4f8d4d47a39cc7b4d1db616aaa_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112069bd2a4f8d4d47a39cc7b4d1db616aaa_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,402 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208751eadd737c46efb0f20194af6dd867_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208751eadd737c46efb0f20194af6dd867_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,403 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120881a29a9c57840d2a36b4d252ca587eb_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120881a29a9c57840d2a36b4d252ca587eb_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,404 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112089bd8445bb55486abe59981ff942db36_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112089bd8445bb55486abe59981ff942db36_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,411 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a54b1e532d274c8db6c6e54532ff2b74_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a54b1e532d274c8db6c6e54532ff2b74_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,415 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a77b76aaac1446afb3aa325a78695835_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a77b76aaac1446afb3aa325a78695835_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,416 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba954c65622c49fe8d526215de8d29ea_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba954c65622c49fe8d526215de8d29ea_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,418 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cdf1ff2be58f493fa66d19119363f212_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cdf1ff2be58f493fa66d19119363f212_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,419 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dbdd76e1f492497cab6844a524d093ae_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dbdd76e1f492497cab6844a524d093ae_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,420 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120de653d39278c494983f707121534bcdc_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120de653d39278c494983f707121534bcdc_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,424 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e3ad3a086e964c19a9d2f1c631ccaa96_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e3ad3a086e964c19a9d2f1c631ccaa96_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,425 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e3aff655022049f0a2a624bd7ead6de4_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e3aff655022049f0a2a624bd7ead6de4_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,426 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120eed6b923e96847d7bdf201d8c6d0c379_a71704901305964843e4454ac34f114c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120eed6b923e96847d7bdf201d8c6d0c379_a71704901305964843e4454ac34f114c 2024-11-20T10:26:10,428 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T10:26:10,431 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:10,433 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T10:26:10,436 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T10:26:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T10:26:10,438 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:10,438 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T10:26:10,438 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732098370438"}]},"ts":"9223372036854775807"} 2024-11-20T10:26:10,442 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T10:26:10,442 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a71704901305964843e4454ac34f114c, NAME => 'TestAcidGuarantees,,1732098339871.a71704901305964843e4454ac34f114c.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T10:26:10,442 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T10:26:10,442 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732098370442"}]},"ts":"9223372036854775807"} 2024-11-20T10:26:10,447 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T10:26:10,448 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:10,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 121 msec 2024-11-20T10:26:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T10:26:10,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T10:26:10,647 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:10,657 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239 (was 239), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=455 (was 549), ProcessCount=11 (was 11), AvailableMemoryMB=8188 (was 8279) 2024-11-20T10:26:10,665 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=239, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=455, ProcessCount=11, AvailableMemoryMB=8187 2024-11-20T10:26:10,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T10:26:10,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T10:26:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:10,670 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T10:26:10,670 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:10,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-20T10:26:10,671 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T10:26:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-20T10:26:10,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742263_1439 (size=959) 2024-11-20T10:26:10,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-20T10:26:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-20T10:26:11,084 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31 2024-11-20T10:26:11,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742264_1440 (size=53) 2024-11-20T10:26:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-20T10:26:11,490 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:26:11,490 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e4730fcd99e6bf6b364225fd77cc4c0d, disabling compactions & flushes 2024-11-20T10:26:11,490 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:11,490 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:11,490 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. after waiting 0 ms 2024-11-20T10:26:11,490 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:11,490 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:11,490 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e4730fcd99e6bf6b364225fd77cc4c0d: Waiting for close lock at 1732098371490Disabling compacts and flushes for region at 1732098371490Disabling writes for close at 1732098371490Writing region close event to WAL at 1732098371490Closed at 1732098371490 2024-11-20T10:26:11,491 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T10:26:11,491 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732098371491"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732098371491"}]},"ts":"1732098371491"} 2024-11-20T10:26:11,493 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T10:26:11,494 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T10:26:11,494 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098371494"}]},"ts":"1732098371494"} 2024-11-20T10:26:11,496 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T10:26:11,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4730fcd99e6bf6b364225fd77cc4c0d, ASSIGN}] 2024-11-20T10:26:11,497 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4730fcd99e6bf6b364225fd77cc4c0d, ASSIGN 2024-11-20T10:26:11,498 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4730fcd99e6bf6b364225fd77cc4c0d, ASSIGN; state=OFFLINE, location=8c533afd331c,35833,1732098238678; forceNewPlan=false, retain=false 2024-11-20T10:26:11,649 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=e4730fcd99e6bf6b364225fd77cc4c0d, regionState=OPENING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:26:11,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4730fcd99e6bf6b364225fd77cc4c0d, ASSIGN because future has completed 2024-11-20T10:26:11,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:26:11,805 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:11,805 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:26:11,806 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:11,806 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:26:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-20T10:26:11,806 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:11,806 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:11,807 INFO [StoreOpener-e4730fcd99e6bf6b364225fd77cc4c0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:11,809 INFO [StoreOpener-e4730fcd99e6bf6b364225fd77cc4c0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4730fcd99e6bf6b364225fd77cc4c0d columnFamilyName A 2024-11-20T10:26:11,809 DEBUG [StoreOpener-e4730fcd99e6bf6b364225fd77cc4c0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:11,809 INFO [StoreOpener-e4730fcd99e6bf6b364225fd77cc4c0d-1 {}] regionserver.HStore(327): Store=e4730fcd99e6bf6b364225fd77cc4c0d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:26:11,809 INFO [StoreOpener-e4730fcd99e6bf6b364225fd77cc4c0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:11,810 INFO [StoreOpener-e4730fcd99e6bf6b364225fd77cc4c0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4730fcd99e6bf6b364225fd77cc4c0d columnFamilyName B 2024-11-20T10:26:11,810 DEBUG [StoreOpener-e4730fcd99e6bf6b364225fd77cc4c0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:11,811 INFO [StoreOpener-e4730fcd99e6bf6b364225fd77cc4c0d-1 {}] regionserver.HStore(327): Store=e4730fcd99e6bf6b364225fd77cc4c0d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:26:11,811 INFO [StoreOpener-e4730fcd99e6bf6b364225fd77cc4c0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:11,812 INFO [StoreOpener-e4730fcd99e6bf6b364225fd77cc4c0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4730fcd99e6bf6b364225fd77cc4c0d columnFamilyName C 2024-11-20T10:26:11,812 DEBUG [StoreOpener-e4730fcd99e6bf6b364225fd77cc4c0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:11,812 INFO [StoreOpener-e4730fcd99e6bf6b364225fd77cc4c0d-1 {}] regionserver.HStore(327): Store=e4730fcd99e6bf6b364225fd77cc4c0d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:26:11,813 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:11,813 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:11,813 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:11,814 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:11,814 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:11,814 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T10:26:11,815 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:11,817 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:26:11,817 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened e4730fcd99e6bf6b364225fd77cc4c0d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62814771, jitterRate=-0.06398697197437286}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T10:26:11,817 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:11,818 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for e4730fcd99e6bf6b364225fd77cc4c0d: Running coprocessor pre-open hook at 1732098371806Writing region info on filesystem at 1732098371806Initializing all the Stores at 1732098371807 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098371807Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098371807Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098371807Cleaning up temporary data from old regions at 1732098371814 (+7 ms)Running coprocessor post-open hooks at 1732098371817 (+3 ms)Region opened successfully at 1732098371818 (+1 ms) 2024-11-20T10:26:11,818 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., pid=118, masterSystemTime=1732098371802 2024-11-20T10:26:11,820 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:11,820 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:11,821 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=e4730fcd99e6bf6b364225fd77cc4c0d, regionState=OPEN, openSeqNum=2, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:26:11,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:26:11,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-20T10:26:11,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 in 172 msec 2024-11-20T10:26:11,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-20T10:26:11,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4730fcd99e6bf6b364225fd77cc4c0d, ASSIGN in 328 msec 2024-11-20T10:26:11,826 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T10:26:11,827 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098371826"}]},"ts":"1732098371826"} 2024-11-20T10:26:11,828 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T10:26:11,828 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T10:26:11,829 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-11-20T10:26:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-20T10:26:12,817 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:12,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c7b275, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,818 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:12,818 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:12,818 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6697beb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:12,820 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14dd9cb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,821 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:12,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:12,823 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T10:26:12,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 8c533afd331c,33867,1732098237476 2024-11-20T10:26:12,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4786a77b 2024-11-20T10:26:12,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T10:26:12,824 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T10:26:12,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fcbf455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,825 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:12,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:12,825 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f5a2f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:12,827 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@483a1a9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc683cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:12,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:12,831 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:12,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:12,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f45eb23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:12,833 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@686acf7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43681357, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,834 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:12,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:12,835 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:12,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:12,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:12,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11361a1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:12,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:12,836 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:12,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7f6307, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39982088, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:12,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:12,841 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc712f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:12,842 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f58402b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4c85de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,843 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:12,844 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:12,844 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b76fe97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:12,845 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53839a07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44ce80f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,846 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:12,846 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:12,846 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4928f9cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:12,847 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:12,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5214e0e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@125833c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,848 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:12,848 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:12,849 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:12,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:12,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8acb8f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:12,851 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:12,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@402410ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea16345, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,852 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:12,852 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:12,853 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1508ae8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:12,854 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@162ceac6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2334b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:12,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:12,857 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:12,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:12,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:12,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54112191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:12,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:12,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:12,858 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5083b37a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d1af8a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,859 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:12,859 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:12,860 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1a3894, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:12,861 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba6b40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:12,862 DEBUG [Thread-1999 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:12,862 DEBUG [Thread-1982 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:12,863 DEBUG [Thread-1997 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:12,863 DEBUG [Thread-1990 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:12,863 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:12,864 DEBUG [Thread-1988 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:12,864 DEBUG [Thread-1993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-20T10:26:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:12,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:12,865 DEBUG [Thread-1986 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:12,865 DEBUG [Thread-1984 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:12,865 DEBUG [Thread-1988 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:12,865 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:12,866 DEBUG [Thread-1995 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:12,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:12,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:12,866 DEBUG [Thread-1984 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T10:26:12,866 DEBUG [Thread-1984 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:12,866 DEBUG [Thread-1988 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:26:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:12,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:12,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:12,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:26:12,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:12,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:12,868 DEBUG [Thread-2001 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:12,868 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:26:12,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:12,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:12,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:12,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:26:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:26:12,872 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:26:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:26:12,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:26:12,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:26:12,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2] 2024-11-20T10:26:12,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:12,879 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:12,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:26:12,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/a4287f4cb66747c5bc58269d221f910d is 50, key is test_row_0/A:col10/1732098372877/Put/seqid=0 2024-11-20T10:26:12,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742265_1441 (size=12001) 2024-11-20T10:26:12,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/a4287f4cb66747c5bc58269d221f910d 2024-11-20T10:26:12,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/ea8f7b2320a74a59bc603cad57330239 is 50, key is test_row_0/B:col10/1732098372877/Put/seqid=0 2024-11-20T10:26:12,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742266_1442 (size=12001) 2024-11-20T10:26:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T10:26:12,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:12,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098432976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098432977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:12,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098432978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:12,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:12,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:12,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:12,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:12,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:12,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1732098432991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36826 deadline: 1732098432992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T10:26:13,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:13,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:13,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:13,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:13,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:13,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:13,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098433096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098433097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098433097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36826 deadline: 1732098433107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1732098433107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T10:26:13,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:13,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:13,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:13,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:13,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:13,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:13,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T10:26:13,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098433307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098433307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098433316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T10:26:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:13,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:13,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:13,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1732098433327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36826 deadline: 1732098433327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/ea8f7b2320a74a59bc603cad57330239 2024-11-20T10:26:13,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/8172da80c83b48f0a6774ae9a19499ce is 50, key is test_row_0/C:col10/1732098372877/Put/seqid=0 2024-11-20T10:26:13,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742267_1443 (size=12001) 2024-11-20T10:26:13,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/8172da80c83b48f0a6774ae9a19499ce 2024-11-20T10:26:13,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/a4287f4cb66747c5bc58269d221f910d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/a4287f4cb66747c5bc58269d221f910d 2024-11-20T10:26:13,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/a4287f4cb66747c5bc58269d221f910d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-20T10:26:13,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/ea8f7b2320a74a59bc603cad57330239 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ea8f7b2320a74a59bc603cad57330239 2024-11-20T10:26:13,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ea8f7b2320a74a59bc603cad57330239, entries=150, sequenceid=12, filesize=11.7 K 2024-11-20T10:26:13,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/8172da80c83b48f0a6774ae9a19499ce as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/8172da80c83b48f0a6774ae9a19499ce 2024-11-20T10:26:13,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/8172da80c83b48f0a6774ae9a19499ce, entries=150, sequenceid=12, filesize=11.7 K 2024-11-20T10:26:13,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for e4730fcd99e6bf6b364225fd77cc4c0d in 532ms, sequenceid=12, compaction requested=false 2024-11-20T10:26:13,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:13,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T10:26:13,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:13,480 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T10:26:13,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/0634cc25f66c4ee9b65e9ffa73c1480f is 50, key is test_row_0/A:col10/1732098372974/Put/seqid=0 2024-11-20T10:26:13,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742268_1444 (size=12001) 2024-11-20T10:26:13,493 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/0634cc25f66c4ee9b65e9ffa73c1480f 2024-11-20T10:26:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T10:26:13,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/a0de538e95e24d6b85429bc1476e9622 is 50, key is test_row_0/B:col10/1732098372974/Put/seqid=0 2024-11-20T10:26:13,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742269_1445 (size=12001) 2024-11-20T10:26:13,506 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/a0de538e95e24d6b85429bc1476e9622 2024-11-20T10:26:13,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/87e06fb8160b4c9084f1bb6f44a132a9 is 50, key is test_row_0/C:col10/1732098372974/Put/seqid=0 2024-11-20T10:26:13,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742270_1446 (size=12001) 2024-11-20T10:26:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:13,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:13,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098433640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098433643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098433643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1732098433647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36826 deadline: 1732098433653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098433756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098433756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098433757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:13,936 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/87e06fb8160b4c9084f1bb6f44a132a9 2024-11-20T10:26:13,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/0634cc25f66c4ee9b65e9ffa73c1480f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/0634cc25f66c4ee9b65e9ffa73c1480f 2024-11-20T10:26:13,948 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/0634cc25f66c4ee9b65e9ffa73c1480f, entries=150, sequenceid=38, filesize=11.7 K 2024-11-20T10:26:13,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/a0de538e95e24d6b85429bc1476e9622 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a0de538e95e24d6b85429bc1476e9622 2024-11-20T10:26:13,954 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a0de538e95e24d6b85429bc1476e9622, entries=150, sequenceid=38, filesize=11.7 K 2024-11-20T10:26:13,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/87e06fb8160b4c9084f1bb6f44a132a9 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/87e06fb8160b4c9084f1bb6f44a132a9 2024-11-20T10:26:13,960 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/87e06fb8160b4c9084f1bb6f44a132a9, entries=150, sequenceid=38, filesize=11.7 K 2024-11-20T10:26:13,960 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e4730fcd99e6bf6b364225fd77cc4c0d in 480ms, sequenceid=38, compaction requested=false 2024-11-20T10:26:13,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:13,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:13,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-20T10:26:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-20T10:26:13,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-20T10:26:13,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0940 sec 2024-11-20T10:26:13,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.1020 sec 2024-11-20T10:26:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:13,969 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:13,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:26:13,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/e9faf344b2db41908c769d1d8ca28a1f is 50, key is test_row_0/A:col10/1732098373967/Put/seqid=0 2024-11-20T10:26:14,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742271_1447 (size=12001) 2024-11-20T10:26:14,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/e9faf344b2db41908c769d1d8ca28a1f 2024-11-20T10:26:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T10:26:14,008 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:14,008 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:14,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-20T10:26:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T10:26:14,012 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:14,013 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:14,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:14,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/d21ff14dca6f45f1b3a6620a91381373 is 50, key is test_row_0/B:col10/1732098373967/Put/seqid=0 2024-11-20T10:26:14,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742272_1448 (size=12001) 2024-11-20T10:26:14,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098434045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098434045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098434051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T10:26:14,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T10:26:14,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:14,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:14,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:14,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:14,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:14,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098434167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1732098434167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36826 deadline: 1732098434167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098434174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098434174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T10:26:14,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:14,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:14,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:14,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:14,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T10:26:14,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098434377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098434396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098434396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/d21ff14dca6f45f1b3a6620a91381373 2024-11-20T10:26:14,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/e2ac80df552c43fc831fb784f9d79a7e is 50, key is test_row_0/C:col10/1732098373967/Put/seqid=0 2024-11-20T10:26:14,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742273_1449 (size=12001) 2024-11-20T10:26:14,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/e2ac80df552c43fc831fb784f9d79a7e 2024-11-20T10:26:14,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/e9faf344b2db41908c769d1d8ca28a1f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/e9faf344b2db41908c769d1d8ca28a1f 2024-11-20T10:26:14,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/e9faf344b2db41908c769d1d8ca28a1f, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T10:26:14,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/d21ff14dca6f45f1b3a6620a91381373 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/d21ff14dca6f45f1b3a6620a91381373 2024-11-20T10:26:14,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/d21ff14dca6f45f1b3a6620a91381373, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T10:26:14,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/e2ac80df552c43fc831fb784f9d79a7e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/e2ac80df552c43fc831fb784f9d79a7e 2024-11-20T10:26:14,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/e2ac80df552c43fc831fb784f9d79a7e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T10:26:14,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e4730fcd99e6bf6b364225fd77cc4c0d in 508ms, sequenceid=49, compaction requested=true 2024-11-20T10:26:14,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:14,478 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:14,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:26:14,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:14,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:26:14,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:14,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:26:14,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:14,478 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:14,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T10:26:14,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:14,480 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T10:26:14,485 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:14,485 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/A is initiating minor compaction (all files) 2024-11-20T10:26:14,485 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/A in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:14,485 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/a4287f4cb66747c5bc58269d221f910d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/0634cc25f66c4ee9b65e9ffa73c1480f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/e9faf344b2db41908c769d1d8ca28a1f] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=35.2 K 2024-11-20T10:26:14,486 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:14,486 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4287f4cb66747c5bc58269d221f910d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732098372871 2024-11-20T10:26:14,486 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/B is initiating minor compaction (all files) 2024-11-20T10:26:14,486 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/B in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:14,486 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ea8f7b2320a74a59bc603cad57330239, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a0de538e95e24d6b85429bc1476e9622, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/d21ff14dca6f45f1b3a6620a91381373] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=35.2 K 2024-11-20T10:26:14,487 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0634cc25f66c4ee9b65e9ffa73c1480f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732098372974 2024-11-20T10:26:14,487 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9faf344b2db41908c769d1d8ca28a1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732098373639 2024-11-20T10:26:14,487 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ea8f7b2320a74a59bc603cad57330239, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732098372871 2024-11-20T10:26:14,488 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a0de538e95e24d6b85429bc1476e9622, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732098372974 2024-11-20T10:26:14,488 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d21ff14dca6f45f1b3a6620a91381373, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732098373639 2024-11-20T10:26:14,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/728e775bb68e49078fb53a7abc70ce31 is 50, key is test_row_0/A:col10/1732098374043/Put/seqid=0 2024-11-20T10:26:14,510 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#A#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:14,511 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/6b56a7bf05784450a66bdfaaabf47b9d is 50, key is test_row_0/A:col10/1732098373967/Put/seqid=0 2024-11-20T10:26:14,516 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#B#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:14,517 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/8c4e0656ac6b4054ae16a2bf42008f9a is 50, key is test_row_0/B:col10/1732098373967/Put/seqid=0 2024-11-20T10:26:14,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742274_1450 (size=12001) 2024-11-20T10:26:14,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742275_1451 (size=12104) 2024-11-20T10:26:14,570 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/6b56a7bf05784450a66bdfaaabf47b9d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/6b56a7bf05784450a66bdfaaabf47b9d 2024-11-20T10:26:14,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742276_1452 (size=12104) 2024-11-20T10:26:14,583 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/A of e4730fcd99e6bf6b364225fd77cc4c0d into 6b56a7bf05784450a66bdfaaabf47b9d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:14,583 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:14,583 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/A, priority=13, startTime=1732098374477; duration=0sec 2024-11-20T10:26:14,584 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:14,584 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:A 2024-11-20T10:26:14,584 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:14,585 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:14,585 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/C is initiating minor compaction (all files) 2024-11-20T10:26:14,585 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/C in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:14,585 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/8172da80c83b48f0a6774ae9a19499ce, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/87e06fb8160b4c9084f1bb6f44a132a9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/e2ac80df552c43fc831fb784f9d79a7e] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=35.2 K 2024-11-20T10:26:14,586 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8172da80c83b48f0a6774ae9a19499ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732098372871 2024-11-20T10:26:14,586 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87e06fb8160b4c9084f1bb6f44a132a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732098372974 2024-11-20T10:26:14,587 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2ac80df552c43fc831fb784f9d79a7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732098373639 2024-11-20T10:26:14,615 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#C#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:14,616 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/7014647fe30546ea8df59fd5af9e7a82 is 50, key is test_row_0/C:col10/1732098373967/Put/seqid=0 2024-11-20T10:26:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T10:26:14,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742277_1453 (size=12104) 2024-11-20T10:26:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:14,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:14,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098434740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098434742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098434745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098434857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098434857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098434867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:14,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:14,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:14,952 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/728e775bb68e49078fb53a7abc70ce31 2024-11-20T10:26:14,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/c94aeee543024d9897cd1fe172991efb is 50, key is test_row_0/B:col10/1732098374043/Put/seqid=0 2024-11-20T10:26:14,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742278_1454 (size=12001) 2024-11-20T10:26:14,984 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/8c4e0656ac6b4054ae16a2bf42008f9a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/8c4e0656ac6b4054ae16a2bf42008f9a 2024-11-20T10:26:14,988 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/B of e4730fcd99e6bf6b364225fd77cc4c0d into 8c4e0656ac6b4054ae16a2bf42008f9a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:14,988 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:14,988 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/B, priority=13, startTime=1732098374478; duration=0sec 2024-11-20T10:26:14,988 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:14,988 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:B 2024-11-20T10:26:15,062 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/7014647fe30546ea8df59fd5af9e7a82 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/7014647fe30546ea8df59fd5af9e7a82 2024-11-20T10:26:15,069 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/C of e4730fcd99e6bf6b364225fd77cc4c0d into 7014647fe30546ea8df59fd5af9e7a82(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:15,069 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:15,069 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/C, priority=13, startTime=1732098374478; duration=0sec 2024-11-20T10:26:15,069 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:15,069 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:C 2024-11-20T10:26:15,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098435067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098435067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:15,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098435086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T10:26:15,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1732098435187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:15,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36826 deadline: 1732098435187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:15,191 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in e4730fcd99e6bf6b364225fd77cc4c0d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:15,191 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_0' in e4730fcd99e6bf6b364225fd77cc4c0d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:15,374 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/c94aeee543024d9897cd1fe172991efb 2024-11-20T10:26:15,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/c08ebf9ec48641fd9c8b781122a85947 is 50, key is test_row_0/C:col10/1732098374043/Put/seqid=0 2024-11-20T10:26:15,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098435377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:15,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742279_1455 (size=12001) 2024-11-20T10:26:15,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098435387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:15,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:15,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098435406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:15,564 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T10:26:15,791 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/c08ebf9ec48641fd9c8b781122a85947 2024-11-20T10:26:15,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/728e775bb68e49078fb53a7abc70ce31 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/728e775bb68e49078fb53a7abc70ce31 2024-11-20T10:26:15,800 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/728e775bb68e49078fb53a7abc70ce31, entries=150, sequenceid=74, filesize=11.7 K 2024-11-20T10:26:15,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/c94aeee543024d9897cd1fe172991efb as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/c94aeee543024d9897cd1fe172991efb 2024-11-20T10:26:15,805 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/c94aeee543024d9897cd1fe172991efb, entries=150, sequenceid=74, filesize=11.7 K 2024-11-20T10:26:15,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/c08ebf9ec48641fd9c8b781122a85947 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c08ebf9ec48641fd9c8b781122a85947 2024-11-20T10:26:15,810 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c08ebf9ec48641fd9c8b781122a85947, entries=150, sequenceid=74, filesize=11.7 K 2024-11-20T10:26:15,810 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e4730fcd99e6bf6b364225fd77cc4c0d in 1330ms, sequenceid=74, compaction requested=false 2024-11-20T10:26:15,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:15,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:15,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-20T10:26:15,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-20T10:26:15,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-20T10:26:15,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7990 sec 2024-11-20T10:26:15,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.8070 sec 2024-11-20T10:26:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:15,900 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:15,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T10:26:15,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/07d58b9f63f94b7e8b6838d0cce338b0 is 50, key is test_row_1/A:col10/1732098375898/Put/seqid=0 2024-11-20T10:26:15,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742280_1456 (size=11997) 2024-11-20T10:26:15,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098435958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:15,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098435959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:15,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098435960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:15,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:15,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:16,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098436076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:16,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098436076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:16,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098436076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:16,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T10:26:16,157 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:16,157 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-20T10:26:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T10:26:16,159 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:16,159 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:16,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T10:26:16,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098436287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:16,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098436288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098436288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:16,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T10:26:16,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:16,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:16,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:16,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:16,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:16,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/07d58b9f63f94b7e8b6838d0cce338b0 2024-11-20T10:26:16,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/debfe9a9db254503b246ef4530ebf6e5 is 50, key is test_row_1/B:col10/1732098375898/Put/seqid=0 2024-11-20T10:26:16,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742281_1457 (size=9657) 2024-11-20T10:26:16,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T10:26:16,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:16,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:16,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:16,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:16,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T10:26:16,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098436606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:16,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098436606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:16,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098436606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:16,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T10:26:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:16,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:16,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/debfe9a9db254503b246ef4530ebf6e5 2024-11-20T10:26:16,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/9ccefffbbace432f96351cbdf16cec34 is 50, key is test_row_1/C:col10/1732098375898/Put/seqid=0 2024-11-20T10:26:16,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742282_1458 (size=9657) 2024-11-20T10:26:16,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/9ccefffbbace432f96351cbdf16cec34 2024-11-20T10:26:16,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/07d58b9f63f94b7e8b6838d0cce338b0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/07d58b9f63f94b7e8b6838d0cce338b0 2024-11-20T10:26:16,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/07d58b9f63f94b7e8b6838d0cce338b0, entries=150, sequenceid=89, filesize=11.7 K 2024-11-20T10:26:16,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/debfe9a9db254503b246ef4530ebf6e5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/debfe9a9db254503b246ef4530ebf6e5 2024-11-20T10:26:16,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/debfe9a9db254503b246ef4530ebf6e5, entries=100, sequenceid=89, filesize=9.4 K 2024-11-20T10:26:16,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/9ccefffbbace432f96351cbdf16cec34 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9ccefffbbace432f96351cbdf16cec34 2024-11-20T10:26:16,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9ccefffbbace432f96351cbdf16cec34, entries=100, sequenceid=89, filesize=9.4 K 2024-11-20T10:26:16,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e4730fcd99e6bf6b364225fd77cc4c0d in 867ms, sequenceid=89, compaction requested=true 2024-11-20T10:26:16,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:16,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:26:16,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:16,767 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:16,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:26:16,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:16,767 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:16,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:26:16,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:16,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T10:26:16,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:16,769 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T10:26:16,771 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36102 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:16,771 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:16,771 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/A is initiating minor compaction (all files) 2024-11-20T10:26:16,771 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/B is initiating minor compaction (all files) 2024-11-20T10:26:16,771 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/A in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:16,771 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/B in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:16,771 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/6b56a7bf05784450a66bdfaaabf47b9d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/728e775bb68e49078fb53a7abc70ce31, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/07d58b9f63f94b7e8b6838d0cce338b0] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=35.3 K 2024-11-20T10:26:16,771 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/8c4e0656ac6b4054ae16a2bf42008f9a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/c94aeee543024d9897cd1fe172991efb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/debfe9a9db254503b246ef4530ebf6e5] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=33.0 K 2024-11-20T10:26:16,771 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b56a7bf05784450a66bdfaaabf47b9d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732098373639 2024-11-20T10:26:16,772 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c4e0656ac6b4054ae16a2bf42008f9a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732098373639 2024-11-20T10:26:16,772 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 728e775bb68e49078fb53a7abc70ce31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732098374042 2024-11-20T10:26:16,772 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07d58b9f63f94b7e8b6838d0cce338b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732098374727 2024-11-20T10:26:16,772 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c94aeee543024d9897cd1fe172991efb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732098374042 2024-11-20T10:26:16,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/2cbf4b1d7c7445d6b9a3248b4b12bd81 is 50, key is test_row_0/A:col10/1732098375948/Put/seqid=0 2024-11-20T10:26:16,775 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting debfe9a9db254503b246ef4530ebf6e5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732098374743 2024-11-20T10:26:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T10:26:16,788 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#A#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:16,788 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/dd75493325f04dcf81ff62bcba0d330b is 50, key is test_row_0/A:col10/1732098374043/Put/seqid=0 2024-11-20T10:26:16,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742283_1459 (size=12001) 2024-11-20T10:26:16,793 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/2cbf4b1d7c7445d6b9a3248b4b12bd81 2024-11-20T10:26:16,798 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#B#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:16,798 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/2b1c372e3c3641cb89bf86c522184ea4 is 50, key is test_row_0/B:col10/1732098374043/Put/seqid=0 2024-11-20T10:26:16,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742284_1460 (size=12207) 2024-11-20T10:26:16,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742285_1461 (size=12207) 2024-11-20T10:26:16,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/22b21ca2b6614eb2906f6b4a7e837099 is 50, key is test_row_0/B:col10/1732098375948/Put/seqid=0 2024-11-20T10:26:16,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742286_1462 (size=12001) 2024-11-20T10:26:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:17,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:17,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098437146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098437149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098437155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,210 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/dd75493325f04dcf81ff62bcba0d330b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/dd75493325f04dcf81ff62bcba0d330b 2024-11-20T10:26:17,214 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/A of e4730fcd99e6bf6b364225fd77cc4c0d into dd75493325f04dcf81ff62bcba0d330b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:17,214 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:17,214 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/A, priority=13, startTime=1732098376767; duration=0sec 2024-11-20T10:26:17,214 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:17,214 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:A 2024-11-20T10:26:17,214 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:17,215 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:17,215 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/C is initiating minor compaction (all files) 2024-11-20T10:26:17,215 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/C in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:17,215 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/7014647fe30546ea8df59fd5af9e7a82, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c08ebf9ec48641fd9c8b781122a85947, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9ccefffbbace432f96351cbdf16cec34] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=33.0 K 2024-11-20T10:26:17,215 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7014647fe30546ea8df59fd5af9e7a82, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732098373639 2024-11-20T10:26:17,215 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c08ebf9ec48641fd9c8b781122a85947, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732098374042 2024-11-20T10:26:17,216 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ccefffbbace432f96351cbdf16cec34, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732098374743 2024-11-20T10:26:17,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36826 deadline: 1732098437206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,216 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_0' in e4730fcd99e6bf6b364225fd77cc4c0d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:17,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1732098437217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,218 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in e4730fcd99e6bf6b364225fd77cc4c0d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,222 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/2b1c372e3c3641cb89bf86c522184ea4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/2b1c372e3c3641cb89bf86c522184ea4 2024-11-20T10:26:17,226 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#C#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:17,226 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/a7f4c9211dce4bb098a863f4bd029698 is 50, key is test_row_0/C:col10/1732098374043/Put/seqid=0 2024-11-20T10:26:17,226 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/B of e4730fcd99e6bf6b364225fd77cc4c0d into 2b1c372e3c3641cb89bf86c522184ea4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:17,226 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:17,226 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/B, priority=13, startTime=1732098376767; duration=0sec 2024-11-20T10:26:17,226 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:17,226 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:B 2024-11-20T10:26:17,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742287_1463 (size=12207) 2024-11-20T10:26:17,242 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/22b21ca2b6614eb2906f6b4a7e837099 2024-11-20T10:26:17,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/58397a0ba6ae461cbfdcd1d151df2c1c is 50, key is test_row_0/C:col10/1732098375948/Put/seqid=0 2024-11-20T10:26:17,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742288_1464 (size=12001) 2024-11-20T10:26:17,258 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/58397a0ba6ae461cbfdcd1d151df2c1c 2024-11-20T10:26:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/2cbf4b1d7c7445d6b9a3248b4b12bd81 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/2cbf4b1d7c7445d6b9a3248b4b12bd81 2024-11-20T10:26:17,265 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/2cbf4b1d7c7445d6b9a3248b4b12bd81, entries=150, sequenceid=113, filesize=11.7 K 2024-11-20T10:26:17,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/22b21ca2b6614eb2906f6b4a7e837099 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/22b21ca2b6614eb2906f6b4a7e837099 2024-11-20T10:26:17,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098437257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098437266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098437266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,272 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/22b21ca2b6614eb2906f6b4a7e837099, entries=150, sequenceid=113, filesize=11.7 K 2024-11-20T10:26:17,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/58397a0ba6ae461cbfdcd1d151df2c1c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/58397a0ba6ae461cbfdcd1d151df2c1c 2024-11-20T10:26:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,279 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/58397a0ba6ae461cbfdcd1d151df2c1c, entries=150, sequenceid=113, filesize=11.7 K 2024-11-20T10:26:17,280 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e4730fcd99e6bf6b364225fd77cc4c0d in 511ms, sequenceid=113, compaction requested=false 2024-11-20T10:26:17,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:17,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:17,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-20T10:26:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-20T10:26:17,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-20T10:26:17,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1230 sec 2024-11-20T10:26:17,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.1270 sec 2024-11-20T10:26:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T10:26:17,297 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:17,297 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-20T10:26:17,299 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T10:26:17,300 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:17,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T10:26:17,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T10:26:17,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:17,452 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T10:26:17,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/121b8342e1324341982b3b0dfe6e0824 is 50, key is test_row_0/A:col10/1732098377143/Put/seqid=0 2024-11-20T10:26:17,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742289_1465 (size=12001) 2024-11-20T10:26:17,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:17,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:17,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098437562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098437577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098437577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T10:26:17,633 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/a7f4c9211dce4bb098a863f4bd029698 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/a7f4c9211dce4bb098a863f4bd029698 2024-11-20T10:26:17,637 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/C of e4730fcd99e6bf6b364225fd77cc4c0d into a7f4c9211dce4bb098a863f4bd029698(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:17,637 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:17,637 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/C, priority=13, startTime=1732098376767; duration=0sec 2024-11-20T10:26:17,637 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:17,637 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:C 2024-11-20T10:26:17,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098437686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098437686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098437696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,869 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/121b8342e1324341982b3b0dfe6e0824 2024-11-20T10:26:17,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/1ac981a0d6c8467c9ee78a5c39fce7d4 is 50, key is test_row_0/B:col10/1732098377143/Put/seqid=0 2024-11-20T10:26:17,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742290_1466 (size=12001) 2024-11-20T10:26:17,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098437896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098437896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098437907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:17,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T10:26:18,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098438206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:18,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098438206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:18,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098438217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:18,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:18,283 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/1ac981a0d6c8467c9ee78a5c39fce7d4 2024-11-20T10:26:18,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/97c7b0b7739e4558bf0e647be4c41c04 is 50, key is test_row_0/C:col10/1732098377143/Put/seqid=0 2024-11-20T10:26:18,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742291_1467 (size=12001) 2024-11-20T10:26:18,293 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/97c7b0b7739e4558bf0e647be4c41c04 2024-11-20T10:26:18,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/121b8342e1324341982b3b0dfe6e0824 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/121b8342e1324341982b3b0dfe6e0824 2024-11-20T10:26:18,300 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/121b8342e1324341982b3b0dfe6e0824, entries=150, sequenceid=127, filesize=11.7 K 2024-11-20T10:26:18,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/1ac981a0d6c8467c9ee78a5c39fce7d4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/1ac981a0d6c8467c9ee78a5c39fce7d4 2024-11-20T10:26:18,303 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/1ac981a0d6c8467c9ee78a5c39fce7d4, entries=150, sequenceid=127, filesize=11.7 K 2024-11-20T10:26:18,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/97c7b0b7739e4558bf0e647be4c41c04 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/97c7b0b7739e4558bf0e647be4c41c04 2024-11-20T10:26:18,307 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/97c7b0b7739e4558bf0e647be4c41c04, entries=150, sequenceid=127, filesize=11.7 K 2024-11-20T10:26:18,308 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e4730fcd99e6bf6b364225fd77cc4c0d in 856ms, sequenceid=127, compaction requested=true 2024-11-20T10:26:18,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:18,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:18,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-20T10:26:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-20T10:26:18,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-20T10:26:18,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0090 sec 2024-11-20T10:26:18,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.0150 sec 2024-11-20T10:26:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T10:26:18,437 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:18,437 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-20T10:26:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T10:26:18,439 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:18,439 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:18,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T10:26:18,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T10:26:18,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:18,592 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T10:26:18,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/89a738c96cc549f282f2123223c85324 is 50, key is test_row_0/A:col10/1732098377560/Put/seqid=0 2024-11-20T10:26:18,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742292_1468 (size=12151) 2024-11-20T10:26:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:18,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:18,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098438743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:18,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:18,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098438744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098438745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T10:26:18,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098438857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:18,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:18,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098438866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:18,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098438868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:19,002 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/89a738c96cc549f282f2123223c85324 2024-11-20T10:26:19,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/a91178a8228d47d28414c74a4a1e09c7 is 50, key is test_row_0/B:col10/1732098377560/Put/seqid=0 2024-11-20T10:26:19,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742293_1469 (size=12151) 2024-11-20T10:26:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T10:26:19,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098439067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:19,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098439076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:19,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098439076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:19,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:19,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098439377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:19,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:19,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098439387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098439387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:19,422 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/a91178a8228d47d28414c74a4a1e09c7 2024-11-20T10:26:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/65efcdeae3de41ec955417be557e63ff is 50, key is test_row_0/C:col10/1732098377560/Put/seqid=0 2024-11-20T10:26:19,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742294_1470 (size=12151) 2024-11-20T10:26:19,441 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/65efcdeae3de41ec955417be557e63ff 2024-11-20T10:26:19,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/89a738c96cc549f282f2123223c85324 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/89a738c96cc549f282f2123223c85324 2024-11-20T10:26:19,449 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/89a738c96cc549f282f2123223c85324, entries=150, sequenceid=152, filesize=11.9 K 2024-11-20T10:26:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/a91178a8228d47d28414c74a4a1e09c7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a91178a8228d47d28414c74a4a1e09c7 2024-11-20T10:26:19,452 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a91178a8228d47d28414c74a4a1e09c7, entries=150, sequenceid=152, filesize=11.9 K 2024-11-20T10:26:19,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/65efcdeae3de41ec955417be557e63ff as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/65efcdeae3de41ec955417be557e63ff 2024-11-20T10:26:19,456 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/65efcdeae3de41ec955417be557e63ff, entries=150, sequenceid=152, filesize=11.9 K 2024-11-20T10:26:19,457 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e4730fcd99e6bf6b364225fd77cc4c0d in 865ms, sequenceid=152, compaction requested=true 2024-11-20T10:26:19,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:19,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:19,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-20T10:26:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-20T10:26:19,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-20T10:26:19,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0210 sec 2024-11-20T10:26:19,463 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.0250 sec 2024-11-20T10:26:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T10:26:19,577 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:19,577 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-20T10:26:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T10:26:19,578 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:19,579 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:19,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T10:26:19,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T10:26:19,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:19,731 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T10:26:19,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/b79952233b4e4531ac50b08ac209810a is 50, key is test_row_0/A:col10/1732098378744/Put/seqid=0 2024-11-20T10:26:19,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742295_1471 (size=12151) 2024-11-20T10:26:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T10:26:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:19,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:19,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:19,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098439975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:19,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098439982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:20,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098439986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:20,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098440097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:20,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:20,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098440109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:20,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098440110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:20,143 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/b79952233b4e4531ac50b08ac209810a 2024-11-20T10:26:20,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/48477790bee94aefa54c5a53027e723c is 50, key is test_row_0/B:col10/1732098378744/Put/seqid=0 2024-11-20T10:26:20,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742296_1472 (size=12151) 2024-11-20T10:26:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T10:26:20,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098440316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:20,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098440326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:20,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098440327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:20,553 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/48477790bee94aefa54c5a53027e723c 2024-11-20T10:26:20,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/1dfb178654b24b709aaebe5d39423740 is 50, key is test_row_0/C:col10/1732098378744/Put/seqid=0 2024-11-20T10:26:20,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742297_1473 (size=12151) 2024-11-20T10:26:20,561 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/1dfb178654b24b709aaebe5d39423740 2024-11-20T10:26:20,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/b79952233b4e4531ac50b08ac209810a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/b79952233b4e4531ac50b08ac209810a 2024-11-20T10:26:20,567 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/b79952233b4e4531ac50b08ac209810a, entries=150, sequenceid=164, filesize=11.9 K 2024-11-20T10:26:20,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/48477790bee94aefa54c5a53027e723c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/48477790bee94aefa54c5a53027e723c 2024-11-20T10:26:20,570 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/48477790bee94aefa54c5a53027e723c, entries=150, sequenceid=164, filesize=11.9 K 2024-11-20T10:26:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/1dfb178654b24b709aaebe5d39423740 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/1dfb178654b24b709aaebe5d39423740 2024-11-20T10:26:20,573 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/1dfb178654b24b709aaebe5d39423740, entries=150, sequenceid=164, filesize=11.9 K 2024-11-20T10:26:20,574 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e4730fcd99e6bf6b364225fd77cc4c0d in 843ms, sequenceid=164, compaction requested=true 2024-11-20T10:26:20,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:20,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:20,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-20T10:26:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-20T10:26:20,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-20T10:26:20,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 996 msec 2024-11-20T10:26:20,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0000 sec 2024-11-20T10:26:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:20,633 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:20,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T10:26:20,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/e7455a40d4cd4c0188ea2573aa73ad19 is 50, key is test_row_0/A:col10/1732098379984/Put/seqid=0 2024-11-20T10:26:20,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742298_1474 (size=14541) 2024-11-20T10:26:20,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098440662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:20,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098440674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:20,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098440675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T10:26:20,717 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:20,717 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-20T10:26:20,719 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T10:26:20,719 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:20,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:20,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098440786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098440786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098440787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:20,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:20,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T10:26:20,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T10:26:20,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:20,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:20,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:21,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098441007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:21,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098441007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:21,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098441007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:21,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T10:26:21,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:21,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:21,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:21,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:21,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:21,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T10:26:21,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/e7455a40d4cd4c0188ea2573aa73ad19 2024-11-20T10:26:21,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/ff4b73fa4a824f3485d5e2ce527cdfc5 is 50, key is test_row_0/B:col10/1732098379984/Put/seqid=0 2024-11-20T10:26:21,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742299_1475 (size=12151) 2024-11-20T10:26:21,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/ff4b73fa4a824f3485d5e2ce527cdfc5 2024-11-20T10:26:21,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/d66c69849b7e4dfa902ac0124aa2f796 is 50, key is test_row_0/C:col10/1732098379984/Put/seqid=0 2024-11-20T10:26:21,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742300_1476 (size=12151) 2024-11-20T10:26:21,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T10:26:21,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:21,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:21,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:21,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:21,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:21,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1732098441237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:21,245 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in e4730fcd99e6bf6b364225fd77cc4c0d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:21,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:21,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36826 deadline: 1732098441256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:21,260 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_0' in e4730fcd99e6bf6b364225fd77cc4c0d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:21,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T10:26:21,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:21,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:21,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:21,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:21,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:21,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:21,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098441328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:21,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098441328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098441328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:21,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T10:26:21,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/d66c69849b7e4dfa902ac0124aa2f796 2024-11-20T10:26:21,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T10:26:21,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:21,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:21,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:21,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:21,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:21,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/e7455a40d4cd4c0188ea2573aa73ad19 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/e7455a40d4cd4c0188ea2573aa73ad19 2024-11-20T10:26:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:21,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/e7455a40d4cd4c0188ea2573aa73ad19, entries=200, sequenceid=189, filesize=14.2 K 2024-11-20T10:26:21,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/ff4b73fa4a824f3485d5e2ce527cdfc5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ff4b73fa4a824f3485d5e2ce527cdfc5 2024-11-20T10:26:21,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ff4b73fa4a824f3485d5e2ce527cdfc5, entries=150, sequenceid=189, filesize=11.9 K 2024-11-20T10:26:21,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/d66c69849b7e4dfa902ac0124aa2f796 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/d66c69849b7e4dfa902ac0124aa2f796 2024-11-20T10:26:21,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/d66c69849b7e4dfa902ac0124aa2f796, entries=150, sequenceid=189, filesize=11.9 K 2024-11-20T10:26:21,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e4730fcd99e6bf6b364225fd77cc4c0d in 865ms, sequenceid=189, compaction requested=true 2024-11-20T10:26:21,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:21,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:26:21,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:21,498 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-20T10:26:21,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:26:21,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:21,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:26:21,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T10:26:21,498 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-20T10:26:21,500 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72662 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-20T10:26:21,500 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/B is initiating minor compaction (all files) 2024-11-20T10:26:21,500 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/B in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:21,500 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/2b1c372e3c3641cb89bf86c522184ea4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/22b21ca2b6614eb2906f6b4a7e837099, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/1ac981a0d6c8467c9ee78a5c39fce7d4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a91178a8228d47d28414c74a4a1e09c7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/48477790bee94aefa54c5a53027e723c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ff4b73fa4a824f3485d5e2ce527cdfc5] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=71.0 K 2024-11-20T10:26:21,501 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 75052 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-20T10:26:21,501 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/A is initiating minor compaction (all files) 2024-11-20T10:26:21,501 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/A in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:21,501 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/dd75493325f04dcf81ff62bcba0d330b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/2cbf4b1d7c7445d6b9a3248b4b12bd81, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/121b8342e1324341982b3b0dfe6e0824, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/89a738c96cc549f282f2123223c85324, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/b79952233b4e4531ac50b08ac209810a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/e7455a40d4cd4c0188ea2573aa73ad19] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=73.3 K 2024-11-20T10:26:21,501 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b1c372e3c3641cb89bf86c522184ea4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732098374043 2024-11-20T10:26:21,501 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd75493325f04dcf81ff62bcba0d330b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732098374043 2024-11-20T10:26:21,501 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 22b21ca2b6614eb2906f6b4a7e837099, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732098375936 2024-11-20T10:26:21,501 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cbf4b1d7c7445d6b9a3248b4b12bd81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732098375936 2024-11-20T10:26:21,501 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ac981a0d6c8467c9ee78a5c39fce7d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732098377142 2024-11-20T10:26:21,502 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 121b8342e1324341982b3b0dfe6e0824, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732098377142 2024-11-20T10:26:21,502 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a91178a8228d47d28414c74a4a1e09c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732098377546 2024-11-20T10:26:21,502 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89a738c96cc549f282f2123223c85324, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732098377546 2024-11-20T10:26:21,502 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting b79952233b4e4531ac50b08ac209810a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732098378733 2024-11-20T10:26:21,503 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 48477790bee94aefa54c5a53027e723c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732098378733 2024-11-20T10:26:21,503 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7455a40d4cd4c0188ea2573aa73ad19, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732098379966 2024-11-20T10:26:21,503 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ff4b73fa4a824f3485d5e2ce527cdfc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732098379966 2024-11-20T10:26:21,519 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#A#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:21,519 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/765fc28ea45e4d3abefc75b6c943ed6f is 50, key is test_row_0/A:col10/1732098379984/Put/seqid=0 2024-11-20T10:26:21,523 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#B#compaction#400 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:21,523 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/366e95b78cdc46a5adde8bd2683a7fe8 is 50, key is test_row_0/B:col10/1732098379984/Put/seqid=0 2024-11-20T10:26:21,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742301_1477 (size=12561) 2024-11-20T10:26:21,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742302_1478 (size=12561) 2024-11-20T10:26:21,542 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/366e95b78cdc46a5adde8bd2683a7fe8 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/366e95b78cdc46a5adde8bd2683a7fe8 2024-11-20T10:26:21,547 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/B of e4730fcd99e6bf6b364225fd77cc4c0d into 366e95b78cdc46a5adde8bd2683a7fe8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:21,547 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:21,547 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/B, priority=10, startTime=1732098381498; duration=0sec 2024-11-20T10:26:21,547 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:21,547 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:B 2024-11-20T10:26:21,548 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-20T10:26:21,549 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72662 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-20T10:26:21,549 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/C is initiating minor compaction (all files) 2024-11-20T10:26:21,549 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/C in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:21,549 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/a7f4c9211dce4bb098a863f4bd029698, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/58397a0ba6ae461cbfdcd1d151df2c1c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/97c7b0b7739e4558bf0e647be4c41c04, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/65efcdeae3de41ec955417be557e63ff, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/1dfb178654b24b709aaebe5d39423740, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/d66c69849b7e4dfa902ac0124aa2f796] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=71.0 K 2024-11-20T10:26:21,549 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a7f4c9211dce4bb098a863f4bd029698, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732098374043 2024-11-20T10:26:21,549 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 58397a0ba6ae461cbfdcd1d151df2c1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732098375936 2024-11-20T10:26:21,551 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 97c7b0b7739e4558bf0e647be4c41c04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732098377142 2024-11-20T10:26:21,552 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 65efcdeae3de41ec955417be557e63ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732098377546 2024-11-20T10:26:21,552 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dfb178654b24b709aaebe5d39423740, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732098378733 2024-11-20T10:26:21,552 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d66c69849b7e4dfa902ac0124aa2f796, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732098379966 2024-11-20T10:26:21,560 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#C#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:21,561 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/9d912396fa0f4e518b78f416dadb678c is 50, key is test_row_0/C:col10/1732098379984/Put/seqid=0 2024-11-20T10:26:21,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742303_1479 (size=12561) 2024-11-20T10:26:21,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T10:26:21,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:21,634 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:26:21,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/2f2c302b8b3a40a1a4969de2c0897329 is 50, key is test_row_0/A:col10/1732098380674/Put/seqid=0 2024-11-20T10:26:21,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742304_1480 (size=12151) 2024-11-20T10:26:21,642 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/2f2c302b8b3a40a1a4969de2c0897329 2024-11-20T10:26:21,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/bd6f0abb28084573a92d06248a0a5dc8 is 50, key is test_row_0/B:col10/1732098380674/Put/seqid=0 2024-11-20T10:26:21,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742305_1481 (size=12151) 2024-11-20T10:26:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:21,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T10:26:21,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098441925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:21,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098441926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:21,940 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/765fc28ea45e4d3abefc75b6c943ed6f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/765fc28ea45e4d3abefc75b6c943ed6f 2024-11-20T10:26:21,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098441929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:21,945 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/A of e4730fcd99e6bf6b364225fd77cc4c0d into 765fc28ea45e4d3abefc75b6c943ed6f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:21,946 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:21,946 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/A, priority=10, startTime=1732098381498; duration=0sec 2024-11-20T10:26:21,946 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:21,946 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:A 2024-11-20T10:26:21,970 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/9d912396fa0f4e518b78f416dadb678c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9d912396fa0f4e518b78f416dadb678c 2024-11-20T10:26:21,977 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/C of e4730fcd99e6bf6b364225fd77cc4c0d into 9d912396fa0f4e518b78f416dadb678c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:21,977 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:21,977 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/C, priority=10, startTime=1732098381498; duration=0sec 2024-11-20T10:26:21,977 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:21,977 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:C 2024-11-20T10:26:22,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098442047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:22,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098442047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:22,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098442047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:22,063 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/bd6f0abb28084573a92d06248a0a5dc8 2024-11-20T10:26:22,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/2651f020aa164a51a364ad725267621e is 50, key is test_row_0/C:col10/1732098380674/Put/seqid=0 2024-11-20T10:26:22,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742306_1482 (size=12151) 2024-11-20T10:26:22,076 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/2651f020aa164a51a364ad725267621e 2024-11-20T10:26:22,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/2f2c302b8b3a40a1a4969de2c0897329 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/2f2c302b8b3a40a1a4969de2c0897329 2024-11-20T10:26:22,086 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/2f2c302b8b3a40a1a4969de2c0897329, entries=150, sequenceid=201, filesize=11.9 K 2024-11-20T10:26:22,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/bd6f0abb28084573a92d06248a0a5dc8 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bd6f0abb28084573a92d06248a0a5dc8 2024-11-20T10:26:22,090 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bd6f0abb28084573a92d06248a0a5dc8, entries=150, sequenceid=201, filesize=11.9 K 2024-11-20T10:26:22,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/2651f020aa164a51a364ad725267621e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/2651f020aa164a51a364ad725267621e 2024-11-20T10:26:22,094 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/2651f020aa164a51a364ad725267621e, entries=150, sequenceid=201, filesize=11.9 K 2024-11-20T10:26:22,095 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e4730fcd99e6bf6b364225fd77cc4c0d in 461ms, sequenceid=201, compaction requested=false 2024-11-20T10:26:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-20T10:26:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-20T10:26:22,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-20T10:26:22,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3780 sec 2024-11-20T10:26:22,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.3820 sec 2024-11-20T10:26:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:22,261 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:22,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T10:26:22,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/ce6fcc8d865f451e85d2f962731ff069 is 50, key is test_row_0/A:col10/1732098381907/Put/seqid=0 2024-11-20T10:26:22,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742307_1483 (size=14541) 2024-11-20T10:26:22,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098442309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:22,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098442309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:22,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098442309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:22,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098442426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:22,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098442427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098442427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:22,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098442636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:22,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098442647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098442647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:22,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/ce6fcc8d865f451e85d2f962731ff069 2024-11-20T10:26:22,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/a4b8ec7c778f4be495c85b35ee50328e is 50, key is test_row_0/B:col10/1732098381907/Put/seqid=0 2024-11-20T10:26:22,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742308_1484 (size=12151) 2024-11-20T10:26:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T10:26:22,867 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:22,867 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-20T10:26:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T10:26:22,869 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:22,869 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:22,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:22,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098442946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:22,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:22,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098442966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:22,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:22,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098442967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:22,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:22,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:22,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T10:26:23,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T10:26:23,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:23,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:23,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:23,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:23,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:23,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/a4b8ec7c778f4be495c85b35ee50328e 2024-11-20T10:26:23,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/085298868216414cab12925799c1afbb is 50, key is test_row_0/C:col10/1732098381907/Put/seqid=0 2024-11-20T10:26:23,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742309_1485 (size=12151) 2024-11-20T10:26:23,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T10:26:23,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:23,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:23,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:23,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:23,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T10:26:23,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T10:26:23,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:23,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:23,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:23,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:23,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:23,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098443466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:23,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098443477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:23,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T10:26:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:23,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:23,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:23,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:23,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:23,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:23,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/085298868216414cab12925799c1afbb 2024-11-20T10:26:23,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098443487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/ce6fcc8d865f451e85d2f962731ff069 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/ce6fcc8d865f451e85d2f962731ff069 2024-11-20T10:26:23,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/ce6fcc8d865f451e85d2f962731ff069, entries=200, sequenceid=229, filesize=14.2 K 2024-11-20T10:26:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T10:26:23,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/a4b8ec7c778f4be495c85b35ee50328e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a4b8ec7c778f4be495c85b35ee50328e 2024-11-20T10:26:23,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a4b8ec7c778f4be495c85b35ee50328e, entries=150, sequenceid=229, filesize=11.9 K 2024-11-20T10:26:23,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/085298868216414cab12925799c1afbb as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/085298868216414cab12925799c1afbb 2024-11-20T10:26:23,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/085298868216414cab12925799c1afbb, entries=150, sequenceid=229, filesize=11.9 K 2024-11-20T10:26:23,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e4730fcd99e6bf6b364225fd77cc4c0d in 1244ms, sequenceid=229, compaction requested=true 2024-11-20T10:26:23,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:23,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:26:23,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:23,506 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:23,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:26:23,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:23,506 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:23,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:26:23,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:23,507 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:23,507 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:23,507 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/B is initiating minor compaction (all files) 2024-11-20T10:26:23,507 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/A is initiating minor compaction (all files) 2024-11-20T10:26:23,507 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/A in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:23,507 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/B in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:23,507 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/366e95b78cdc46a5adde8bd2683a7fe8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bd6f0abb28084573a92d06248a0a5dc8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a4b8ec7c778f4be495c85b35ee50328e] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=36.0 K 2024-11-20T10:26:23,507 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/765fc28ea45e4d3abefc75b6c943ed6f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/2f2c302b8b3a40a1a4969de2c0897329, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/ce6fcc8d865f451e85d2f962731ff069] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=38.3 K 2024-11-20T10:26:23,507 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 765fc28ea45e4d3abefc75b6c943ed6f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732098379966 2024-11-20T10:26:23,507 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 366e95b78cdc46a5adde8bd2683a7fe8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732098379966 2024-11-20T10:26:23,507 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f2c302b8b3a40a1a4969de2c0897329, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732098380649 2024-11-20T10:26:23,507 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting bd6f0abb28084573a92d06248a0a5dc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732098380649 2024-11-20T10:26:23,508 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a4b8ec7c778f4be495c85b35ee50328e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732098381907 2024-11-20T10:26:23,508 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce6fcc8d865f451e85d2f962731ff069, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732098381907 2024-11-20T10:26:23,516 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#B#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:23,516 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#A#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:23,516 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/2c71f15cdf6e44baa781a561f14a200e is 50, key is test_row_0/B:col10/1732098381907/Put/seqid=0 2024-11-20T10:26:23,516 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/074a63b0660c48fda6e3cabc8cca08c0 is 50, key is test_row_0/A:col10/1732098381907/Put/seqid=0 2024-11-20T10:26:23,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742310_1486 (size=12663) 2024-11-20T10:26:23,532 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/074a63b0660c48fda6e3cabc8cca08c0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/074a63b0660c48fda6e3cabc8cca08c0 2024-11-20T10:26:23,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742311_1487 (size=12663) 2024-11-20T10:26:23,542 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/A of e4730fcd99e6bf6b364225fd77cc4c0d into 074a63b0660c48fda6e3cabc8cca08c0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:23,542 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:23,542 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/A, priority=13, startTime=1732098383506; duration=0sec 2024-11-20T10:26:23,542 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:23,542 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:A 2024-11-20T10:26:23,542 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:23,544 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:23,545 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/C is initiating minor compaction (all files) 2024-11-20T10:26:23,545 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/C in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:23,545 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9d912396fa0f4e518b78f416dadb678c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/2651f020aa164a51a364ad725267621e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/085298868216414cab12925799c1afbb] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=36.0 K 2024-11-20T10:26:23,545 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d912396fa0f4e518b78f416dadb678c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732098379966 2024-11-20T10:26:23,545 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2651f020aa164a51a364ad725267621e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732098380649 2024-11-20T10:26:23,545 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 085298868216414cab12925799c1afbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732098381907 2024-11-20T10:26:23,558 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:23,558 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/c1aae30ca13f41f4b59d6a20a7b28ac1 is 50, key is test_row_0/C:col10/1732098381907/Put/seqid=0 2024-11-20T10:26:23,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742312_1488 (size=12663) 2024-11-20T10:26:23,569 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/c1aae30ca13f41f4b59d6a20a7b28ac1 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c1aae30ca13f41f4b59d6a20a7b28ac1 2024-11-20T10:26:23,576 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/C of e4730fcd99e6bf6b364225fd77cc4c0d into c1aae30ca13f41f4b59d6a20a7b28ac1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:23,576 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:23,576 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/C, priority=13, startTime=1732098383506; duration=0sec 2024-11-20T10:26:23,576 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:23,576 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:C 2024-11-20T10:26:23,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T10:26:23,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:23,633 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-20T10:26:23,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/717bbcceba6f4987b4563d93e04ffdb5 is 50, key is test_row_0/A:col10/1732098382272/Put/seqid=0 2024-11-20T10:26:23,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742313_1489 (size=12151) 2024-11-20T10:26:23,956 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/2c71f15cdf6e44baa781a561f14a200e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/2c71f15cdf6e44baa781a561f14a200e 2024-11-20T10:26:23,963 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/B of e4730fcd99e6bf6b364225fd77cc4c0d into 2c71f15cdf6e44baa781a561f14a200e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:23,963 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:23,963 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/B, priority=13, startTime=1732098383506; duration=0sec 2024-11-20T10:26:23,963 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:23,963 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:B 2024-11-20T10:26:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T10:26:24,071 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/717bbcceba6f4987b4563d93e04ffdb5 2024-11-20T10:26:24,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/efa3a537424c48dab93a9e57e1644bd5 is 50, key is test_row_0/B:col10/1732098382272/Put/seqid=0 2024-11-20T10:26:24,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742314_1490 (size=12151) 2024-11-20T10:26:24,143 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/efa3a537424c48dab93a9e57e1644bd5 2024-11-20T10:26:24,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/4bf8e234714c41e99b9ad7e097585c64 is 50, key is test_row_0/C:col10/1732098382272/Put/seqid=0 2024-11-20T10:26:24,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742315_1491 (size=12151) 2024-11-20T10:26:24,178 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/4bf8e234714c41e99b9ad7e097585c64 2024-11-20T10:26:24,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/717bbcceba6f4987b4563d93e04ffdb5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/717bbcceba6f4987b4563d93e04ffdb5 2024-11-20T10:26:24,193 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/717bbcceba6f4987b4563d93e04ffdb5, entries=150, sequenceid=241, filesize=11.9 K 2024-11-20T10:26:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/efa3a537424c48dab93a9e57e1644bd5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/efa3a537424c48dab93a9e57e1644bd5 2024-11-20T10:26:24,196 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/efa3a537424c48dab93a9e57e1644bd5, entries=150, sequenceid=241, filesize=11.9 K 2024-11-20T10:26:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/4bf8e234714c41e99b9ad7e097585c64 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/4bf8e234714c41e99b9ad7e097585c64 2024-11-20T10:26:24,200 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/4bf8e234714c41e99b9ad7e097585c64, entries=150, sequenceid=241, filesize=11.9 K 2024-11-20T10:26:24,201 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for e4730fcd99e6bf6b364225fd77cc4c0d in 568ms, sequenceid=241, compaction requested=false 2024-11-20T10:26:24,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:24,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:24,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-20T10:26:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-20T10:26:24,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-20T10:26:24,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3330 sec 2024-11-20T10:26:24,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.3370 sec 2024-11-20T10:26:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:24,492 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:24,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:26:24,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/1a2e76b8f8594d3396448aaea1f0b650 is 50, key is test_row_0/A:col10/1732098384485/Put/seqid=0 2024-11-20T10:26:24,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742316_1492 (size=14541) 2024-11-20T10:26:24,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098444564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:24,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098444564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:24,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098444568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:24,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098444676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:24,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098444676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:24,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098444676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:24,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098444887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098444887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098444887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:24,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/1a2e76b8f8594d3396448aaea1f0b650 2024-11-20T10:26:24,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/076877ab74b54fcba3413cf594457da7 is 50, key is test_row_0/B:col10/1732098384485/Put/seqid=0 2024-11-20T10:26:24,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742317_1493 (size=12151) 2024-11-20T10:26:24,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/076877ab74b54fcba3413cf594457da7 2024-11-20T10:26:24,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/7e11b76514c54fad8f138661f204e9c2 is 50, key is test_row_0/C:col10/1732098384485/Put/seqid=0 2024-11-20T10:26:24,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742318_1494 (size=12151) 2024-11-20T10:26:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T10:26:25,017 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:25,017 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-20T10:26:25,019 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:25,020 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:25,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T10:26:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T10:26:25,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T10:26:25,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:25,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:25,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:25,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:25,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:25,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098445206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:25,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:25,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098445206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:25,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098445207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:25,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T10:26:25,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:25,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:25,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:25,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:25,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:25,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/7e11b76514c54fad8f138661f204e9c2 2024-11-20T10:26:25,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/1a2e76b8f8594d3396448aaea1f0b650 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/1a2e76b8f8594d3396448aaea1f0b650 2024-11-20T10:26:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T10:26:25,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/1a2e76b8f8594d3396448aaea1f0b650, entries=200, sequenceid=253, filesize=14.2 K 2024-11-20T10:26:25,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/076877ab74b54fcba3413cf594457da7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/076877ab74b54fcba3413cf594457da7 2024-11-20T10:26:25,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/076877ab74b54fcba3413cf594457da7, entries=150, sequenceid=253, filesize=11.9 K 2024-11-20T10:26:25,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/7e11b76514c54fad8f138661f204e9c2 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/7e11b76514c54fad8f138661f204e9c2 2024-11-20T10:26:25,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/7e11b76514c54fad8f138661f204e9c2, entries=150, sequenceid=253, filesize=11.9 K 2024-11-20T10:26:25,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e4730fcd99e6bf6b364225fd77cc4c0d in 853ms, sequenceid=253, compaction requested=true 2024-11-20T10:26:25,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:25,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:26:25,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:25,346 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:25,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:26:25,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:25,346 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:25,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:26:25,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:25,346 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:25,346 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/A is initiating minor compaction (all files) 2024-11-20T10:26:25,347 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/A in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:25,347 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/074a63b0660c48fda6e3cabc8cca08c0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/717bbcceba6f4987b4563d93e04ffdb5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/1a2e76b8f8594d3396448aaea1f0b650] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=38.4 K 2024-11-20T10:26:25,347 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 074a63b0660c48fda6e3cabc8cca08c0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732098381907 2024-11-20T10:26:25,347 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 717bbcceba6f4987b4563d93e04ffdb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732098382271 2024-11-20T10:26:25,347 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a2e76b8f8594d3396448aaea1f0b650, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732098384478 2024-11-20T10:26:25,348 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:25,348 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/B is initiating minor compaction (all files) 2024-11-20T10:26:25,348 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/B in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:25,348 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/2c71f15cdf6e44baa781a561f14a200e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/efa3a537424c48dab93a9e57e1644bd5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/076877ab74b54fcba3413cf594457da7] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=36.1 K 2024-11-20T10:26:25,349 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c71f15cdf6e44baa781a561f14a200e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732098381907 2024-11-20T10:26:25,349 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting efa3a537424c48dab93a9e57e1644bd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732098382271 2024-11-20T10:26:25,349 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 076877ab74b54fcba3413cf594457da7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732098384485 2024-11-20T10:26:25,355 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#A#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:25,356 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/4ebcd8e49fc2436ab2bad02802b30262 is 50, key is test_row_0/A:col10/1732098384485/Put/seqid=0 2024-11-20T10:26:25,359 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#B#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:25,360 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/8b63aa9d8ec04a62aad525c4d5263565 is 50, key is test_row_0/B:col10/1732098384485/Put/seqid=0 2024-11-20T10:26:25,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742319_1495 (size=12765) 2024-11-20T10:26:25,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742320_1496 (size=12765) 2024-11-20T10:26:25,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T10:26:25,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:25,478 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T10:26:25,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/ceae926f0c0c4fcfaf0d15f143943b2b is 50, key is test_row_0/A:col10/1732098384566/Put/seqid=0 2024-11-20T10:26:25,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742321_1497 (size=12301) 2024-11-20T10:26:25,491 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/ceae926f0c0c4fcfaf0d15f143943b2b 2024-11-20T10:26:25,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/ee54d4931316447081ff286edb1a9f97 is 50, key is test_row_0/B:col10/1732098384566/Put/seqid=0 2024-11-20T10:26:25,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742322_1498 (size=12301) 2024-11-20T10:26:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T10:26:25,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:25,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098445749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:25,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:25,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098445752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:25,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:25,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:25,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098445753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:25,780 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/4ebcd8e49fc2436ab2bad02802b30262 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/4ebcd8e49fc2436ab2bad02802b30262 2024-11-20T10:26:25,784 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/8b63aa9d8ec04a62aad525c4d5263565 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/8b63aa9d8ec04a62aad525c4d5263565 2024-11-20T10:26:25,786 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/A of e4730fcd99e6bf6b364225fd77cc4c0d into 4ebcd8e49fc2436ab2bad02802b30262(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:25,786 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:25,786 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/A, priority=13, startTime=1732098385345; duration=0sec 2024-11-20T10:26:25,786 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:25,786 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:A 2024-11-20T10:26:25,787 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:25,788 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:25,788 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/C is initiating minor compaction (all files) 2024-11-20T10:26:25,788 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/C in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:25,788 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c1aae30ca13f41f4b59d6a20a7b28ac1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/4bf8e234714c41e99b9ad7e097585c64, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/7e11b76514c54fad8f138661f204e9c2] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=36.1 K 2024-11-20T10:26:25,788 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1aae30ca13f41f4b59d6a20a7b28ac1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732098381907 2024-11-20T10:26:25,789 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bf8e234714c41e99b9ad7e097585c64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732098382271 2024-11-20T10:26:25,789 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e11b76514c54fad8f138661f204e9c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732098384485 2024-11-20T10:26:25,791 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/B of e4730fcd99e6bf6b364225fd77cc4c0d into 8b63aa9d8ec04a62aad525c4d5263565(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:25,791 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:25,791 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/B, priority=13, startTime=1732098385345; duration=0sec 2024-11-20T10:26:25,791 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:25,791 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:B 2024-11-20T10:26:25,795 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#C#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:25,795 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/3f2e9e667d6a45c48ac043799a2891ea is 50, key is test_row_0/C:col10/1732098384485/Put/seqid=0 2024-11-20T10:26:25,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742323_1499 (size=12765) 2024-11-20T10:26:25,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098445866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:25,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098445866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098445867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:25,923 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/ee54d4931316447081ff286edb1a9f97 2024-11-20T10:26:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/aa6cbc12ac334a41b4cf783fc5c180f9 is 50, key is test_row_0/C:col10/1732098384566/Put/seqid=0 2024-11-20T10:26:25,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742324_1500 (size=12301) 2024-11-20T10:26:26,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098446076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:26,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098446076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:26,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098446086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T10:26:26,213 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/3f2e9e667d6a45c48ac043799a2891ea as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/3f2e9e667d6a45c48ac043799a2891ea 2024-11-20T10:26:26,216 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/C of e4730fcd99e6bf6b364225fd77cc4c0d into 3f2e9e667d6a45c48ac043799a2891ea(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:26,216 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:26,216 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/C, priority=13, startTime=1732098385346; duration=0sec 2024-11-20T10:26:26,216 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:26,216 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:C 2024-11-20T10:26:26,336 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/aa6cbc12ac334a41b4cf783fc5c180f9 2024-11-20T10:26:26,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/ceae926f0c0c4fcfaf0d15f143943b2b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/ceae926f0c0c4fcfaf0d15f143943b2b 2024-11-20T10:26:26,342 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/ceae926f0c0c4fcfaf0d15f143943b2b, entries=150, sequenceid=278, filesize=12.0 K 2024-11-20T10:26:26,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/ee54d4931316447081ff286edb1a9f97 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ee54d4931316447081ff286edb1a9f97 2024-11-20T10:26:26,346 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ee54d4931316447081ff286edb1a9f97, entries=150, sequenceid=278, filesize=12.0 K 2024-11-20T10:26:26,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/aa6cbc12ac334a41b4cf783fc5c180f9 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/aa6cbc12ac334a41b4cf783fc5c180f9 2024-11-20T10:26:26,350 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/aa6cbc12ac334a41b4cf783fc5c180f9, entries=150, sequenceid=278, filesize=12.0 K 2024-11-20T10:26:26,351 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e4730fcd99e6bf6b364225fd77cc4c0d in 872ms, sequenceid=278, compaction requested=false 2024-11-20T10:26:26,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:26,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:26,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-20T10:26:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-20T10:26:26,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-20T10:26:26,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3320 sec 2024-11-20T10:26:26,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.3360 sec 2024-11-20T10:26:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:26,389 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:26,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T10:26:26,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/3199a41fe0834743bab0bd41251ea4c1 is 50, key is test_row_0/A:col10/1732098386388/Put/seqid=0 2024-11-20T10:26:26,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742325_1501 (size=14741) 2024-11-20T10:26:26,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/3199a41fe0834743bab0bd41251ea4c1 2024-11-20T10:26:26,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/cb2c68550e1a458c852abced58ad015e is 50, key is test_row_0/B:col10/1732098386388/Put/seqid=0 2024-11-20T10:26:26,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742326_1502 (size=12301) 2024-11-20T10:26:26,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/cb2c68550e1a458c852abced58ad015e 2024-11-20T10:26:26,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/b0ef3f635d8a49b7af7f0863169bd227 is 50, key is test_row_0/C:col10/1732098386388/Put/seqid=0 2024-11-20T10:26:26,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742327_1503 (size=12301) 2024-11-20T10:26:26,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098446446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:26,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098446453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:26,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098446454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:26,535 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T10:26:26,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098446566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:26,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098446576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098446577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:26,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098446776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:26,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098446786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:26,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098446786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:26,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/b0ef3f635d8a49b7af7f0863169bd227 2024-11-20T10:26:26,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/3199a41fe0834743bab0bd41251ea4c1 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/3199a41fe0834743bab0bd41251ea4c1 2024-11-20T10:26:26,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/3199a41fe0834743bab0bd41251ea4c1, entries=200, sequenceid=293, filesize=14.4 K 2024-11-20T10:26:26,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/cb2c68550e1a458c852abced58ad015e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/cb2c68550e1a458c852abced58ad015e 2024-11-20T10:26:26,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/cb2c68550e1a458c852abced58ad015e, entries=150, sequenceid=293, filesize=12.0 K 2024-11-20T10:26:26,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/b0ef3f635d8a49b7af7f0863169bd227 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/b0ef3f635d8a49b7af7f0863169bd227 2024-11-20T10:26:26,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/b0ef3f635d8a49b7af7f0863169bd227, entries=150, sequenceid=293, filesize=12.0 K 2024-11-20T10:26:26,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e4730fcd99e6bf6b364225fd77cc4c0d in 458ms, sequenceid=293, compaction requested=true 2024-11-20T10:26:26,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:26,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:26:26,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:26,848 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:26,848 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:26,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:26:26,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:26,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:26:26,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:26,848 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:26,848 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:26,849 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/B is initiating minor compaction (all files) 2024-11-20T10:26:26,849 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/A is initiating minor compaction (all files) 2024-11-20T10:26:26,849 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/B in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:26,849 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/A in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:26,849 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/8b63aa9d8ec04a62aad525c4d5263565, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ee54d4931316447081ff286edb1a9f97, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/cb2c68550e1a458c852abced58ad015e] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=36.5 K 2024-11-20T10:26:26,849 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/4ebcd8e49fc2436ab2bad02802b30262, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/ceae926f0c0c4fcfaf0d15f143943b2b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/3199a41fe0834743bab0bd41251ea4c1] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=38.9 K 2024-11-20T10:26:26,849 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b63aa9d8ec04a62aad525c4d5263565, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732098384485 2024-11-20T10:26:26,849 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ebcd8e49fc2436ab2bad02802b30262, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732098384485 2024-11-20T10:26:26,849 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ee54d4931316447081ff286edb1a9f97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732098384562 2024-11-20T10:26:26,849 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceae926f0c0c4fcfaf0d15f143943b2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732098384562 2024-11-20T10:26:26,849 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3199a41fe0834743bab0bd41251ea4c1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732098385735 2024-11-20T10:26:26,849 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting cb2c68550e1a458c852abced58ad015e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732098385735 2024-11-20T10:26:26,858 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#A#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:26,858 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#B#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:26,858 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/b7613cf7c3ca4432bcbbe4a73354abb4 is 50, key is test_row_0/B:col10/1732098386388/Put/seqid=0 2024-11-20T10:26:26,858 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/3c57e307e47b48779d0c7a23642395eb is 50, key is test_row_0/A:col10/1732098386388/Put/seqid=0 2024-11-20T10:26:26,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742329_1505 (size=13017) 2024-11-20T10:26:26,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742328_1504 (size=13017) 2024-11-20T10:26:26,877 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/b7613cf7c3ca4432bcbbe4a73354abb4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/b7613cf7c3ca4432bcbbe4a73354abb4 2024-11-20T10:26:26,881 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/B of e4730fcd99e6bf6b364225fd77cc4c0d into b7613cf7c3ca4432bcbbe4a73354abb4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:26,881 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:26,881 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/B, priority=13, startTime=1732098386848; duration=0sec 2024-11-20T10:26:26,881 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:26,881 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:B 2024-11-20T10:26:26,881 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:26,882 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:26,882 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/C is initiating minor compaction (all files) 2024-11-20T10:26:26,882 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/C in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:26,882 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/3f2e9e667d6a45c48ac043799a2891ea, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/aa6cbc12ac334a41b4cf783fc5c180f9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/b0ef3f635d8a49b7af7f0863169bd227] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=36.5 K 2024-11-20T10:26:26,882 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f2e9e667d6a45c48ac043799a2891ea, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732098384485 2024-11-20T10:26:26,882 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting aa6cbc12ac334a41b4cf783fc5c180f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732098384562 2024-11-20T10:26:26,882 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b0ef3f635d8a49b7af7f0863169bd227, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732098385735 2024-11-20T10:26:26,890 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#C#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:26,891 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/19dc9a6f069c4671ab1672227c6854f6 is 50, key is test_row_0/C:col10/1732098386388/Put/seqid=0 2024-11-20T10:26:26,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742330_1506 (size=13017) 2024-11-20T10:26:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:27,099 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:27,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T10:26:27,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/4b0ea2b5c52b4eda8fcb4075cc087383 is 50, key is test_row_0/A:col10/1732098386453/Put/seqid=0 2024-11-20T10:26:27,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742331_1507 (size=14741) 2024-11-20T10:26:27,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098447121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:27,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098447122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:27,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098447127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T10:26:27,167 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:27,167 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-20T10:26:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T10:26:27,169 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:27,169 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:27,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:27,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098447236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098447236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:27,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098447246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:27,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:27,276 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/3c57e307e47b48779d0c7a23642395eb as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/3c57e307e47b48779d0c7a23642395eb 2024-11-20T10:26:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T10:26:27,282 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/A of e4730fcd99e6bf6b364225fd77cc4c0d into 3c57e307e47b48779d0c7a23642395eb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:27,282 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:27,282 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/A, priority=13, startTime=1732098386848; duration=0sec 2024-11-20T10:26:27,282 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:27,282 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:A 2024-11-20T10:26:27,301 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/19dc9a6f069c4671ab1672227c6854f6 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/19dc9a6f069c4671ab1672227c6854f6 2024-11-20T10:26:27,305 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/C of e4730fcd99e6bf6b364225fd77cc4c0d into 19dc9a6f069c4671ab1672227c6854f6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:27,305 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:27,305 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/C, priority=13, startTime=1732098386848; duration=0sec 2024-11-20T10:26:27,305 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:27,305 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:C 2024-11-20T10:26:27,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T10:26:27,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:27,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:27,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:27,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:27,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:27,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098447446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:27,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:27,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098447446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:27,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:27,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098447456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:27,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T10:26:27,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:27,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:27,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:27,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:27,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T10:26:27,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/4b0ea2b5c52b4eda8fcb4075cc087383 2024-11-20T10:26:27,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/80ca160bceb64e19a3c2c00f14ba5453 is 50, key is test_row_0/B:col10/1732098386453/Put/seqid=0 2024-11-20T10:26:27,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742332_1508 (size=12301) 2024-11-20T10:26:27,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T10:26:27,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:27,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:27,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:27,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:27,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:27,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098447756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:27,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098447756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:27,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098447767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:27,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:27,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:27,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T10:26:27,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:27,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:27,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:27,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:27,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T10:26:27,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/80ca160bceb64e19a3c2c00f14ba5453 2024-11-20T10:26:27,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/371652e13bd74bfca567aa5363ca233c is 50, key is test_row_0/C:col10/1732098386453/Put/seqid=0 2024-11-20T10:26:27,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T10:26:27,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:27,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:27,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:27,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:27,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:27,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742333_1509 (size=12301) 2024-11-20T10:26:27,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/371652e13bd74bfca567aa5363ca233c 2024-11-20T10:26:27,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/4b0ea2b5c52b4eda8fcb4075cc087383 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/4b0ea2b5c52b4eda8fcb4075cc087383 2024-11-20T10:26:27,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/4b0ea2b5c52b4eda8fcb4075cc087383, entries=200, sequenceid=319, filesize=14.4 K 2024-11-20T10:26:27,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/80ca160bceb64e19a3c2c00f14ba5453 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/80ca160bceb64e19a3c2c00f14ba5453 2024-11-20T10:26:27,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/80ca160bceb64e19a3c2c00f14ba5453, entries=150, sequenceid=319, filesize=12.0 K 2024-11-20T10:26:27,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/371652e13bd74bfca567aa5363ca233c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/371652e13bd74bfca567aa5363ca233c 2024-11-20T10:26:27,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/371652e13bd74bfca567aa5363ca233c, entries=150, sequenceid=319, filesize=12.0 K 2024-11-20T10:26:27,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e4730fcd99e6bf6b364225fd77cc4c0d in 867ms, sequenceid=319, compaction requested=false 2024-11-20T10:26:27,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:28,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T10:26:28,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:28,084 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:26:28,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/054b2565301d4367a5609c360f7f99dc is 50, key is test_row_0/A:col10/1732098387120/Put/seqid=0 2024-11-20T10:26:28,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742334_1510 (size=12301) 2024-11-20T10:26:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:28,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T10:26:28,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098448321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:28,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098448322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098448323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:28,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098448436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:28,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098448436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:28,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098448437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:28,496 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/054b2565301d4367a5609c360f7f99dc 2024-11-20T10:26:28,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/54d7ca5a45ab42bf804b3c3abb94e23c is 50, key is test_row_0/B:col10/1732098387120/Put/seqid=0 2024-11-20T10:26:28,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742335_1511 (size=12301) 2024-11-20T10:26:28,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098448646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:28,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098448656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098448657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:28,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:28,913 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/54d7ca5a45ab42bf804b3c3abb94e23c 2024-11-20T10:26:28,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/701e6c284ef04673b00a1ecc807102b1 is 50, key is test_row_0/C:col10/1732098387120/Put/seqid=0 2024-11-20T10:26:28,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742336_1512 (size=12301) 2024-11-20T10:26:28,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098448957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:28,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:28,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:28,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098448966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:28,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:28,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098448976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T10:26:29,321 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/701e6c284ef04673b00a1ecc807102b1 2024-11-20T10:26:29,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/054b2565301d4367a5609c360f7f99dc as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/054b2565301d4367a5609c360f7f99dc 2024-11-20T10:26:29,327 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/054b2565301d4367a5609c360f7f99dc, entries=150, sequenceid=332, filesize=12.0 K 2024-11-20T10:26:29,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/54d7ca5a45ab42bf804b3c3abb94e23c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/54d7ca5a45ab42bf804b3c3abb94e23c 2024-11-20T10:26:29,334 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/54d7ca5a45ab42bf804b3c3abb94e23c, entries=150, sequenceid=332, filesize=12.0 K 2024-11-20T10:26:29,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/701e6c284ef04673b00a1ecc807102b1 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/701e6c284ef04673b00a1ecc807102b1 2024-11-20T10:26:29,339 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/701e6c284ef04673b00a1ecc807102b1, entries=150, sequenceid=332, filesize=12.0 K 2024-11-20T10:26:29,340 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e4730fcd99e6bf6b364225fd77cc4c0d in 1256ms, sequenceid=332, compaction requested=true 2024-11-20T10:26:29,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:29,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:29,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-20T10:26:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-20T10:26:29,342 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-20T10:26:29,342 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1720 sec 2024-11-20T10:26:29,343 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.1750 sec 2024-11-20T10:26:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:29,479 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:29,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T10:26:29,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/219b4960b1574ce4833afd0cd80222d7 is 50, key is test_row_0/A:col10/1732098388322/Put/seqid=0 2024-11-20T10:26:29,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742337_1513 (size=14741) 2024-11-20T10:26:29,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/219b4960b1574ce4833afd0cd80222d7 2024-11-20T10:26:29,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/bfd06a5d26184701b18d842e4a7a5307 is 50, key is test_row_0/B:col10/1732098388322/Put/seqid=0 2024-11-20T10:26:29,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742338_1514 (size=12301) 2024-11-20T10:26:29,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098449504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:29,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098449505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:29,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098449512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:29,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098449616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:29,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098449626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:29,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098449637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:29,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:29,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098449836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:29,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:29,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098449846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:29,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098449846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:29,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/bfd06a5d26184701b18d842e4a7a5307 2024-11-20T10:26:29,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/c561714debdf4cc8be7bccb218e1d1c9 is 50, key is test_row_0/C:col10/1732098388322/Put/seqid=0 2024-11-20T10:26:29,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742339_1515 (size=12301) 2024-11-20T10:26:30,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098450147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:30,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098450167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:30,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098450167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:30,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:30,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:30,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/c561714debdf4cc8be7bccb218e1d1c9 2024-11-20T10:26:30,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/219b4960b1574ce4833afd0cd80222d7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/219b4960b1574ce4833afd0cd80222d7 2024-11-20T10:26:30,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/219b4960b1574ce4833afd0cd80222d7, entries=200, sequenceid=358, filesize=14.4 K 2024-11-20T10:26:30,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/bfd06a5d26184701b18d842e4a7a5307 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bfd06a5d26184701b18d842e4a7a5307 2024-11-20T10:26:30,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bfd06a5d26184701b18d842e4a7a5307, entries=150, sequenceid=358, filesize=12.0 K 2024-11-20T10:26:30,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/c561714debdf4cc8be7bccb218e1d1c9 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c561714debdf4cc8be7bccb218e1d1c9 2024-11-20T10:26:30,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c561714debdf4cc8be7bccb218e1d1c9, entries=150, sequenceid=358, filesize=12.0 K 2024-11-20T10:26:30,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e4730fcd99e6bf6b364225fd77cc4c0d in 873ms, sequenceid=358, compaction requested=true 2024-11-20T10:26:30,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:30,352 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:26:30,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:26:30,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:30,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:26:30,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:30,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:26:30,353 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:26:30,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:30,353 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:26:30,353 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/A is initiating minor compaction (all files) 2024-11-20T10:26:30,353 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:26:30,353 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/B is initiating minor compaction (all files) 2024-11-20T10:26:30,353 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/A in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:30,353 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/B in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:30,354 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/b7613cf7c3ca4432bcbbe4a73354abb4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/80ca160bceb64e19a3c2c00f14ba5453, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/54d7ca5a45ab42bf804b3c3abb94e23c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bfd06a5d26184701b18d842e4a7a5307] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=48.8 K 2024-11-20T10:26:30,354 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/3c57e307e47b48779d0c7a23642395eb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/4b0ea2b5c52b4eda8fcb4075cc087383, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/054b2565301d4367a5609c360f7f99dc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/219b4960b1574ce4833afd0cd80222d7] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=53.5 K 2024-11-20T10:26:30,354 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b7613cf7c3ca4432bcbbe4a73354abb4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732098385735 2024-11-20T10:26:30,354 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c57e307e47b48779d0c7a23642395eb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732098385735 2024-11-20T10:26:30,354 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b0ea2b5c52b4eda8fcb4075cc087383, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732098386443 2024-11-20T10:26:30,354 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 80ca160bceb64e19a3c2c00f14ba5453, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732098386452 2024-11-20T10:26:30,354 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 54d7ca5a45ab42bf804b3c3abb94e23c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732098387120 2024-11-20T10:26:30,354 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 054b2565301d4367a5609c360f7f99dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732098387120 2024-11-20T10:26:30,355 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 219b4960b1574ce4833afd0cd80222d7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732098388298 2024-11-20T10:26:30,355 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting bfd06a5d26184701b18d842e4a7a5307, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732098388321 2024-11-20T10:26:30,362 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#B#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:30,363 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/95cbd2dc028442b894a4feff13924009 is 50, key is test_row_0/B:col10/1732098388322/Put/seqid=0 2024-11-20T10:26:30,370 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#A#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:30,370 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/16fe5aefb9de48aea42be6dbd76c61c0 is 50, key is test_row_0/A:col10/1732098388322/Put/seqid=0 2024-11-20T10:26:30,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742340_1516 (size=13153) 2024-11-20T10:26:30,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742341_1517 (size=13153) 2024-11-20T10:26:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:30,672 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:30,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:26:30,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/af618ca0948749238f2c6a70a68ed770 is 50, key is test_row_0/A:col10/1732098390671/Put/seqid=0 2024-11-20T10:26:30,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742342_1518 (size=14741) 2024-11-20T10:26:30,783 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/95cbd2dc028442b894a4feff13924009 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/95cbd2dc028442b894a4feff13924009 2024-11-20T10:26:30,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098450776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:30,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098450785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:30,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098450787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:30,794 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/16fe5aefb9de48aea42be6dbd76c61c0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/16fe5aefb9de48aea42be6dbd76c61c0 2024-11-20T10:26:30,794 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/B of e4730fcd99e6bf6b364225fd77cc4c0d into 95cbd2dc028442b894a4feff13924009(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:30,794 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:30,794 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/B, priority=12, startTime=1732098390353; duration=0sec 2024-11-20T10:26:30,794 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:30,794 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:B 2024-11-20T10:26:30,794 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:26:30,796 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:26:30,796 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/C is initiating minor compaction (all files) 2024-11-20T10:26:30,796 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/C in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:30,796 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/19dc9a6f069c4671ab1672227c6854f6, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/371652e13bd74bfca567aa5363ca233c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/701e6c284ef04673b00a1ecc807102b1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c561714debdf4cc8be7bccb218e1d1c9] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=48.8 K 2024-11-20T10:26:30,796 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 19dc9a6f069c4671ab1672227c6854f6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732098385735 2024-11-20T10:26:30,796 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 371652e13bd74bfca567aa5363ca233c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732098386452 2024-11-20T10:26:30,797 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 701e6c284ef04673b00a1ecc807102b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732098387120 2024-11-20T10:26:30,797 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c561714debdf4cc8be7bccb218e1d1c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732098388321 2024-11-20T10:26:30,799 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/A of e4730fcd99e6bf6b364225fd77cc4c0d into 16fe5aefb9de48aea42be6dbd76c61c0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:30,799 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:30,799 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/A, priority=12, startTime=1732098390352; duration=0sec 2024-11-20T10:26:30,799 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:30,799 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:A 2024-11-20T10:26:30,805 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#C#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:30,806 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/fe56a1886867430facfadb434eec0ca7 is 50, key is test_row_0/C:col10/1732098388322/Put/seqid=0 2024-11-20T10:26:30,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742343_1519 (size=13153) 2024-11-20T10:26:30,820 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/fe56a1886867430facfadb434eec0ca7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/fe56a1886867430facfadb434eec0ca7 2024-11-20T10:26:30,824 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/C of e4730fcd99e6bf6b364225fd77cc4c0d into fe56a1886867430facfadb434eec0ca7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:30,824 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:30,824 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/C, priority=12, startTime=1732098390353; duration=0sec 2024-11-20T10:26:30,824 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:30,824 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:C 2024-11-20T10:26:30,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098450896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098450896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:30,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098450896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:31,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/af618ca0948749238f2c6a70a68ed770 2024-11-20T10:26:31,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/bda141f5eeea4881857d8fb4af8f47da is 50, key is test_row_0/B:col10/1732098390671/Put/seqid=0 2024-11-20T10:26:31,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742344_1520 (size=12301) 2024-11-20T10:26:31,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/bda141f5eeea4881857d8fb4af8f47da 2024-11-20T10:26:31,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/22b64d972d444a5c9b0542d40d3fdd34 is 50, key is test_row_0/C:col10/1732098390671/Put/seqid=0 2024-11-20T10:26:31,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742345_1521 (size=12301) 2024-11-20T10:26:31,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098451106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:31,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:31,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098451106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:31,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:31,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:31,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098451106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:31,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:31,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:31,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36826 deadline: 1732098451277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:31,285 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_0' in e4730fcd99e6bf6b364225fd77cc4c0d of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18293 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:31,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T10:26:31,337 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:31,337 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-20T10:26:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T10:26:31,339 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:31,339 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:31,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:31,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1732098451357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:31,361 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in e4730fcd99e6bf6b364225fd77cc4c0d of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18370 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:31,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:31,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098451416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:31,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:31,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098451416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:31,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:31,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098451427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T10:26:31,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T10:26:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:31,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:31,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:31,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/22b64d972d444a5c9b0542d40d3fdd34 2024-11-20T10:26:31,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/af618ca0948749238f2c6a70a68ed770 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/af618ca0948749238f2c6a70a68ed770 2024-11-20T10:26:31,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/af618ca0948749238f2c6a70a68ed770, entries=200, sequenceid=369, filesize=14.4 K 2024-11-20T10:26:31,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/bda141f5eeea4881857d8fb4af8f47da as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bda141f5eeea4881857d8fb4af8f47da 2024-11-20T10:26:31,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bda141f5eeea4881857d8fb4af8f47da, entries=150, sequenceid=369, filesize=12.0 K 2024-11-20T10:26:31,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/22b64d972d444a5c9b0542d40d3fdd34 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/22b64d972d444a5c9b0542d40d3fdd34 2024-11-20T10:26:31,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/22b64d972d444a5c9b0542d40d3fdd34, entries=150, sequenceid=369, filesize=12.0 K 2024-11-20T10:26:31,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for e4730fcd99e6bf6b364225fd77cc4c0d in 843ms, sequenceid=369, compaction requested=false 2024-11-20T10:26:31,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:31,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T10:26:31,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:31,644 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T10:26:31,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/58d37690e484422da0998cd99966bf56 is 50, key is test_row_0/A:col10/1732098390765/Put/seqid=0 2024-11-20T10:26:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T10:26:31,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742346_1522 (size=12301) 2024-11-20T10:26:31,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:31,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:31,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098451945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:31,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098451946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:31,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098451949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:31,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T10:26:32,059 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/58d37690e484422da0998cd99966bf56 2024-11-20T10:26:32,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/3ee199868e3449dda7e1a5791f39b5fb is 50, key is test_row_0/B:col10/1732098390765/Put/seqid=0 2024-11-20T10:26:32,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742347_1523 (size=12301) 2024-11-20T10:26:32,069 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/3ee199868e3449dda7e1a5791f39b5fb 2024-11-20T10:26:32,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098452067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098452067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:32,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098452067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:32,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/9826f45592ca42eeaff247ea5c2d1fdc is 50, key is test_row_0/C:col10/1732098390765/Put/seqid=0 2024-11-20T10:26:32,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742348_1524 (size=12301) 2024-11-20T10:26:32,084 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/9826f45592ca42eeaff247ea5c2d1fdc 2024-11-20T10:26:32,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/58d37690e484422da0998cd99966bf56 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/58d37690e484422da0998cd99966bf56 2024-11-20T10:26:32,090 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/58d37690e484422da0998cd99966bf56, entries=150, sequenceid=398, filesize=12.0 K 2024-11-20T10:26:32,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/3ee199868e3449dda7e1a5791f39b5fb as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/3ee199868e3449dda7e1a5791f39b5fb 2024-11-20T10:26:32,095 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/3ee199868e3449dda7e1a5791f39b5fb, entries=150, sequenceid=398, filesize=12.0 K 2024-11-20T10:26:32,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/9826f45592ca42eeaff247ea5c2d1fdc as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9826f45592ca42eeaff247ea5c2d1fdc 2024-11-20T10:26:32,103 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9826f45592ca42eeaff247ea5c2d1fdc, entries=150, sequenceid=398, filesize=12.0 K 2024-11-20T10:26:32,103 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e4730fcd99e6bf6b364225fd77cc4c0d in 459ms, sequenceid=398, compaction requested=true 2024-11-20T10:26:32,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:32,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:32,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-20T10:26:32,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-20T10:26:32,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-20T10:26:32,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 765 msec 2024-11-20T10:26:32,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 769 msec 2024-11-20T10:26:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:32,292 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:32,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:26:32,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/66b802cea35a4dc0b96809537198e3da is 50, key is test_row_0/A:col10/1732098392291/Put/seqid=0 2024-11-20T10:26:32,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742349_1525 (size=17181) 2024-11-20T10:26:32,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/66b802cea35a4dc0b96809537198e3da 2024-11-20T10:26:32,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/4a29c3a11fab4946b75ff8aa217a50f0 is 50, key is test_row_0/B:col10/1732098392291/Put/seqid=0 2024-11-20T10:26:32,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742350_1526 (size=12301) 2024-11-20T10:26:32,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098452370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:32,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098452374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:32,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098452381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:32,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T10:26:32,477 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:32,477 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-20T10:26:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T10:26:32,479 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:32,480 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:32,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:32,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098452487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:32,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098452496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098452496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T10:26:32,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T10:26:32,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:32,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:32,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:32,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:32,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:32,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36836 deadline: 1732098452697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:32,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36868 deadline: 1732098452716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:32,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:32,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1732098452716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 , the old value is region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., hostname=8c533afd331c,35833,1732098238678, seqNum=2 because the exception is null or not the one we care about 2024-11-20T10:26:32,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/4a29c3a11fab4946b75ff8aa217a50f0 2024-11-20T10:26:32,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/2073d640ee0f466f85590c5bde4f0566 is 50, key is test_row_0/C:col10/1732098392291/Put/seqid=0 2024-11-20T10:26:32,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742351_1527 (size=12301) 2024-11-20T10:26:32,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/2073d640ee0f466f85590c5bde4f0566 2024-11-20T10:26:32,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/66b802cea35a4dc0b96809537198e3da as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/66b802cea35a4dc0b96809537198e3da 2024-11-20T10:26:32,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/66b802cea35a4dc0b96809537198e3da, entries=250, sequenceid=410, filesize=16.8 K 2024-11-20T10:26:32,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/4a29c3a11fab4946b75ff8aa217a50f0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/4a29c3a11fab4946b75ff8aa217a50f0 2024-11-20T10:26:32,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/4a29c3a11fab4946b75ff8aa217a50f0, entries=150, sequenceid=410, filesize=12.0 K 2024-11-20T10:26:32,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/2073d640ee0f466f85590c5bde4f0566 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/2073d640ee0f466f85590c5bde4f0566 2024-11-20T10:26:32,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/2073d640ee0f466f85590c5bde4f0566, entries=150, sequenceid=410, filesize=12.0 K 2024-11-20T10:26:32,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e4730fcd99e6bf6b364225fd77cc4c0d in 470ms, sequenceid=410, compaction requested=true 2024-11-20T10:26:32,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:32,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:26:32,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:32,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:26:32,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:32,762 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:26:32,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4730fcd99e6bf6b364225fd77cc4c0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:26:32,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:32,762 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:26:32,764 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57376 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:26:32,764 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:26:32,765 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/B is initiating minor compaction (all files) 2024-11-20T10:26:32,765 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/A is initiating minor compaction (all files) 2024-11-20T10:26:32,765 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/A in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:32,765 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/B in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:32,765 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/16fe5aefb9de48aea42be6dbd76c61c0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/af618ca0948749238f2c6a70a68ed770, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/58d37690e484422da0998cd99966bf56, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/66b802cea35a4dc0b96809537198e3da] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=56.0 K 2024-11-20T10:26:32,765 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/95cbd2dc028442b894a4feff13924009, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bda141f5eeea4881857d8fb4af8f47da, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/3ee199868e3449dda7e1a5791f39b5fb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/4a29c3a11fab4946b75ff8aa217a50f0] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=48.9 K 2024-11-20T10:26:32,765 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 95cbd2dc028442b894a4feff13924009, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732098388321 2024-11-20T10:26:32,765 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16fe5aefb9de48aea42be6dbd76c61c0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732098388321 2024-11-20T10:26:32,768 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting af618ca0948749238f2c6a70a68ed770, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732098389503 2024-11-20T10:26:32,768 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting bda141f5eeea4881857d8fb4af8f47da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732098389503 2024-11-20T10:26:32,768 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ee199868e3449dda7e1a5791f39b5fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732098390749 2024-11-20T10:26:32,768 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58d37690e484422da0998cd99966bf56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732098390749 2024-11-20T10:26:32,768 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a29c3a11fab4946b75ff8aa217a50f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732098391948 2024-11-20T10:26:32,768 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66b802cea35a4dc0b96809537198e3da, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732098391933 2024-11-20T10:26:32,775 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#B#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:32,775 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/b20a245a284b45d2976b890dad006e67 is 50, key is test_row_0/B:col10/1732098392291/Put/seqid=0 2024-11-20T10:26:32,775 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#A#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:32,776 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/dbad60aebec04af68caa716bd5988d02 is 50, key is test_row_0/A:col10/1732098392291/Put/seqid=0 2024-11-20T10:26:32,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742353_1529 (size=13289) 2024-11-20T10:26:32,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742352_1528 (size=13289) 2024-11-20T10:26:32,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T10:26:32,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:32,785 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T10:26:32,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/81642b31ad414b9e87d35f21ac8bbcde is 50, key is test_row_0/A:col10/1732098392372/Put/seqid=0 2024-11-20T10:26:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T10:26:32,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742354_1530 (size=12301) 2024-11-20T10:26:32,870 INFO [Thread-1999 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1999. 2024-11-20T10:26:32,870 DEBUG [Thread-1999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:32,870 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:32,870 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:32,870 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:32,871 INFO [Thread-1993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1993. 2024-11-20T10:26:32,871 DEBUG [Thread-1993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:32,871 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:32,871 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:32,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:32,871 INFO [Thread-2001 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2001. 2024-11-20T10:26:32,871 DEBUG [Thread-2001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:32,871 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:32,872 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:32,872 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:32,878 INFO [Thread-1995 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1995. 2024-11-20T10:26:32,878 DEBUG [Thread-1995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:32,878 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:32,878 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:32,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:32,881 INFO [Thread-1997 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1997. 2024-11-20T10:26:32,881 DEBUG [Thread-1997 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:32,881 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:32,881 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:32,881 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:33,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. as already flushing 2024-11-20T10:26:33,018 INFO [Thread-1984 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1984. 2024-11-20T10:26:33,018 DEBUG [Thread-1984 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:33,018 DEBUG [Thread-1984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:33,018 DEBUG [Thread-1984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:33,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:33,038 INFO [Thread-1986 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1986. 2024-11-20T10:26:33,038 INFO [Thread-1988 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1988. 2024-11-20T10:26:33,038 DEBUG [Thread-1986 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:33,038 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:33,038 DEBUG [Thread-1988 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:33,038 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:33,038 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:33,038 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:33,038 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:33,038 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:33,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T10:26:33,183 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/dbad60aebec04af68caa716bd5988d02 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/dbad60aebec04af68caa716bd5988d02 2024-11-20T10:26:33,185 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/b20a245a284b45d2976b890dad006e67 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/b20a245a284b45d2976b890dad006e67 2024-11-20T10:26:33,187 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/A of e4730fcd99e6bf6b364225fd77cc4c0d into dbad60aebec04af68caa716bd5988d02(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:33,187 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:33,187 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/A, priority=12, startTime=1732098392762; duration=0sec 2024-11-20T10:26:33,187 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:33,187 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:A 2024-11-20T10:26:33,187 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:26:33,188 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:26:33,188 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): e4730fcd99e6bf6b364225fd77cc4c0d/C is initiating minor compaction (all files) 2024-11-20T10:26:33,188 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4730fcd99e6bf6b364225fd77cc4c0d/C in TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:33,188 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/fe56a1886867430facfadb434eec0ca7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/22b64d972d444a5c9b0542d40d3fdd34, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9826f45592ca42eeaff247ea5c2d1fdc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/2073d640ee0f466f85590c5bde4f0566] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp, totalSize=48.9 K 2024-11-20T10:26:33,188 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe56a1886867430facfadb434eec0ca7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732098388321 2024-11-20T10:26:33,189 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22b64d972d444a5c9b0542d40d3fdd34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732098389503 2024-11-20T10:26:33,189 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9826f45592ca42eeaff247ea5c2d1fdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732098390749 2024-11-20T10:26:33,189 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2073d640ee0f466f85590c5bde4f0566, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732098391948 2024-11-20T10:26:33,190 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/B of e4730fcd99e6bf6b364225fd77cc4c0d into b20a245a284b45d2976b890dad006e67(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:33,190 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:33,190 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/B, priority=12, startTime=1732098392762; duration=0sec 2024-11-20T10:26:33,190 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:33,190 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:B 2024-11-20T10:26:33,194 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4730fcd99e6bf6b364225fd77cc4c0d#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:33,194 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/d90e31570fef4257b427950eb7183fc3 is 50, key is test_row_0/C:col10/1732098392291/Put/seqid=0 2024-11-20T10:26:33,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742355_1531 (size=13289) 2024-11-20T10:26:33,198 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/81642b31ad414b9e87d35f21ac8bbcde 2024-11-20T10:26:33,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/a4619d546e9a4ddfa135878671b210ea is 50, key is test_row_0/B:col10/1732098392372/Put/seqid=0 2024-11-20T10:26:33,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742356_1532 (size=12301) 2024-11-20T10:26:33,601 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/d90e31570fef4257b427950eb7183fc3 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/d90e31570fef4257b427950eb7183fc3 2024-11-20T10:26:33,604 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4730fcd99e6bf6b364225fd77cc4c0d/C of e4730fcd99e6bf6b364225fd77cc4c0d into d90e31570fef4257b427950eb7183fc3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:33,604 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:33,604 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., storeName=e4730fcd99e6bf6b364225fd77cc4c0d/C, priority=12, startTime=1732098392762; duration=0sec 2024-11-20T10:26:33,605 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:33,605 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4730fcd99e6bf6b364225fd77cc4c0d:C 2024-11-20T10:26:33,607 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/a4619d546e9a4ddfa135878671b210ea 2024-11-20T10:26:33,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/97fba9d03b0d4c8490289495e257e124 is 50, key is test_row_0/C:col10/1732098392372/Put/seqid=0 2024-11-20T10:26:33,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742357_1533 (size=12301) 2024-11-20T10:26:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T10:26:34,016 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/97fba9d03b0d4c8490289495e257e124 2024-11-20T10:26:34,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/81642b31ad414b9e87d35f21ac8bbcde as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/81642b31ad414b9e87d35f21ac8bbcde 2024-11-20T10:26:34,023 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/81642b31ad414b9e87d35f21ac8bbcde, entries=150, sequenceid=434, filesize=12.0 K 2024-11-20T10:26:34,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/a4619d546e9a4ddfa135878671b210ea as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a4619d546e9a4ddfa135878671b210ea 2024-11-20T10:26:34,031 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a4619d546e9a4ddfa135878671b210ea, entries=150, sequenceid=434, filesize=12.0 K 2024-11-20T10:26:34,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/97fba9d03b0d4c8490289495e257e124 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/97fba9d03b0d4c8490289495e257e124 2024-11-20T10:26:34,034 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/97fba9d03b0d4c8490289495e257e124, entries=150, sequenceid=434, filesize=12.0 K 2024-11-20T10:26:34,035 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=20.13 KB/20610 for e4730fcd99e6bf6b364225fd77cc4c0d in 1250ms, sequenceid=434, compaction requested=false 2024-11-20T10:26:34,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for e4730fcd99e6bf6b364225fd77cc4c0d: 2024-11-20T10:26:34,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:34,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-20T10:26:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-20T10:26:34,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-20T10:26:34,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5560 sec 2024-11-20T10:26:34,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.5610 sec 2024-11-20T10:26:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T10:26:34,627 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:41,368 INFO [Thread-1982 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1982. 2024-11-20T10:26:41,368 DEBUG [Thread-1982 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:41,368 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:41,368 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:41,368 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:41,408 INFO [Thread-1990 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1990. 2024-11-20T10:26:41,408 DEBUG [Thread-1990 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:26:41,408 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:41,408 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:41,408 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1297 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3890 rows 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1301 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3903 rows 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1310 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3929 rows 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1302 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3901 rows 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1303 2024-11-20T10:26:41,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3909 rows 2024-11-20T10:26:41,408 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T10:26:41,408 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T10:26:41,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:41,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:41,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:26:41,409 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T10:26:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T10:26:41,412 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098401411"}]},"ts":"1732098401411"} 2024-11-20T10:26:41,413 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T10:26:41,413 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T10:26:41,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T10:26:41,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4730fcd99e6bf6b364225fd77cc4c0d, UNASSIGN}] 2024-11-20T10:26:41,415 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4730fcd99e6bf6b364225fd77cc4c0d, UNASSIGN 2024-11-20T10:26:41,415 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=e4730fcd99e6bf6b364225fd77cc4c0d, regionState=CLOSING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:26:41,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4730fcd99e6bf6b364225fd77cc4c0d, UNASSIGN because future has completed 2024-11-20T10:26:41,416 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T10:26:41,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:26:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T10:26:41,569 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:41,569 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T10:26:41,569 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing e4730fcd99e6bf6b364225fd77cc4c0d, disabling compactions & flushes 2024-11-20T10:26:41,569 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:41,569 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:41,569 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. after waiting 0 ms 2024-11-20T10:26:41,569 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:41,569 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(2902): Flushing e4730fcd99e6bf6b364225fd77cc4c0d 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T10:26:41,572 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/16c7fbc1b3ae4dcb8aad6a6542feb81b is 50, key is test_row_0/A:col10/1732098401367/Put/seqid=0 2024-11-20T10:26:41,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742358_1534 (size=12301) 2024-11-20T10:26:41,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T10:26:41,978 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/16c7fbc1b3ae4dcb8aad6a6542feb81b 2024-11-20T10:26:41,984 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/48e9c903b7534dfbab2a37cfbc0ec4bf is 50, key is test_row_0/B:col10/1732098401367/Put/seqid=0 2024-11-20T10:26:41,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742359_1535 (size=12301) 2024-11-20T10:26:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T10:26:42,388 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/48e9c903b7534dfbab2a37cfbc0ec4bf 2024-11-20T10:26:42,393 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/a43cba35da3a47eb9bdec2a3d5df5dc8 is 50, key is test_row_0/C:col10/1732098401367/Put/seqid=0 2024-11-20T10:26:42,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742360_1536 (size=12301) 2024-11-20T10:26:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T10:26:42,797 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/a43cba35da3a47eb9bdec2a3d5df5dc8 2024-11-20T10:26:42,800 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/A/16c7fbc1b3ae4dcb8aad6a6542feb81b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/16c7fbc1b3ae4dcb8aad6a6542feb81b 2024-11-20T10:26:42,803 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/16c7fbc1b3ae4dcb8aad6a6542feb81b, entries=150, sequenceid=445, filesize=12.0 K 2024-11-20T10:26:42,803 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/B/48e9c903b7534dfbab2a37cfbc0ec4bf as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/48e9c903b7534dfbab2a37cfbc0ec4bf 2024-11-20T10:26:42,807 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/48e9c903b7534dfbab2a37cfbc0ec4bf, entries=150, sequenceid=445, filesize=12.0 K 2024-11-20T10:26:42,807 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/.tmp/C/a43cba35da3a47eb9bdec2a3d5df5dc8 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/a43cba35da3a47eb9bdec2a3d5df5dc8 2024-11-20T10:26:42,811 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/a43cba35da3a47eb9bdec2a3d5df5dc8, entries=150, sequenceid=445, filesize=12.0 K 2024-11-20T10:26:42,811 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e4730fcd99e6bf6b364225fd77cc4c0d in 1242ms, sequenceid=445, compaction requested=true 2024-11-20T10:26:42,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/a4287f4cb66747c5bc58269d221f910d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/0634cc25f66c4ee9b65e9ffa73c1480f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/6b56a7bf05784450a66bdfaaabf47b9d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/e9faf344b2db41908c769d1d8ca28a1f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/728e775bb68e49078fb53a7abc70ce31, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/dd75493325f04dcf81ff62bcba0d330b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/07d58b9f63f94b7e8b6838d0cce338b0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/2cbf4b1d7c7445d6b9a3248b4b12bd81, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/121b8342e1324341982b3b0dfe6e0824, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/89a738c96cc549f282f2123223c85324, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/b79952233b4e4531ac50b08ac209810a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/e7455a40d4cd4c0188ea2573aa73ad19, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/765fc28ea45e4d3abefc75b6c943ed6f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/2f2c302b8b3a40a1a4969de2c0897329, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/ce6fcc8d865f451e85d2f962731ff069, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/074a63b0660c48fda6e3cabc8cca08c0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/717bbcceba6f4987b4563d93e04ffdb5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/1a2e76b8f8594d3396448aaea1f0b650, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/4ebcd8e49fc2436ab2bad02802b30262, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/ceae926f0c0c4fcfaf0d15f143943b2b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/3199a41fe0834743bab0bd41251ea4c1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/3c57e307e47b48779d0c7a23642395eb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/4b0ea2b5c52b4eda8fcb4075cc087383, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/054b2565301d4367a5609c360f7f99dc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/219b4960b1574ce4833afd0cd80222d7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/16fe5aefb9de48aea42be6dbd76c61c0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/af618ca0948749238f2c6a70a68ed770, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/58d37690e484422da0998cd99966bf56, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/66b802cea35a4dc0b96809537198e3da] to archive 2024-11-20T10:26:42,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:26:42,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/a4287f4cb66747c5bc58269d221f910d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/a4287f4cb66747c5bc58269d221f910d 2024-11-20T10:26:42,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/0634cc25f66c4ee9b65e9ffa73c1480f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/0634cc25f66c4ee9b65e9ffa73c1480f 2024-11-20T10:26:42,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/6b56a7bf05784450a66bdfaaabf47b9d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/6b56a7bf05784450a66bdfaaabf47b9d 2024-11-20T10:26:42,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/e9faf344b2db41908c769d1d8ca28a1f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/e9faf344b2db41908c769d1d8ca28a1f 2024-11-20T10:26:42,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/728e775bb68e49078fb53a7abc70ce31 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/728e775bb68e49078fb53a7abc70ce31 2024-11-20T10:26:42,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/dd75493325f04dcf81ff62bcba0d330b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/dd75493325f04dcf81ff62bcba0d330b 2024-11-20T10:26:42,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/07d58b9f63f94b7e8b6838d0cce338b0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/07d58b9f63f94b7e8b6838d0cce338b0 2024-11-20T10:26:42,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/2cbf4b1d7c7445d6b9a3248b4b12bd81 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/2cbf4b1d7c7445d6b9a3248b4b12bd81 2024-11-20T10:26:42,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/121b8342e1324341982b3b0dfe6e0824 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/121b8342e1324341982b3b0dfe6e0824 2024-11-20T10:26:42,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/89a738c96cc549f282f2123223c85324 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/89a738c96cc549f282f2123223c85324 2024-11-20T10:26:42,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/b79952233b4e4531ac50b08ac209810a to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/b79952233b4e4531ac50b08ac209810a 2024-11-20T10:26:42,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/e7455a40d4cd4c0188ea2573aa73ad19 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/e7455a40d4cd4c0188ea2573aa73ad19 2024-11-20T10:26:42,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/765fc28ea45e4d3abefc75b6c943ed6f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/765fc28ea45e4d3abefc75b6c943ed6f 2024-11-20T10:26:42,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/2f2c302b8b3a40a1a4969de2c0897329 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/2f2c302b8b3a40a1a4969de2c0897329 2024-11-20T10:26:42,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/ce6fcc8d865f451e85d2f962731ff069 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/ce6fcc8d865f451e85d2f962731ff069 2024-11-20T10:26:42,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/074a63b0660c48fda6e3cabc8cca08c0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/074a63b0660c48fda6e3cabc8cca08c0 2024-11-20T10:26:42,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/717bbcceba6f4987b4563d93e04ffdb5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/717bbcceba6f4987b4563d93e04ffdb5 2024-11-20T10:26:42,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/1a2e76b8f8594d3396448aaea1f0b650 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/1a2e76b8f8594d3396448aaea1f0b650 2024-11-20T10:26:42,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/4ebcd8e49fc2436ab2bad02802b30262 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/4ebcd8e49fc2436ab2bad02802b30262 2024-11-20T10:26:42,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/ceae926f0c0c4fcfaf0d15f143943b2b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/ceae926f0c0c4fcfaf0d15f143943b2b 2024-11-20T10:26:42,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/3199a41fe0834743bab0bd41251ea4c1 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/3199a41fe0834743bab0bd41251ea4c1 2024-11-20T10:26:42,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/3c57e307e47b48779d0c7a23642395eb to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/3c57e307e47b48779d0c7a23642395eb 2024-11-20T10:26:42,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/4b0ea2b5c52b4eda8fcb4075cc087383 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/4b0ea2b5c52b4eda8fcb4075cc087383 2024-11-20T10:26:42,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/054b2565301d4367a5609c360f7f99dc to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/054b2565301d4367a5609c360f7f99dc 2024-11-20T10:26:42,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/219b4960b1574ce4833afd0cd80222d7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/219b4960b1574ce4833afd0cd80222d7 2024-11-20T10:26:42,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/16fe5aefb9de48aea42be6dbd76c61c0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/16fe5aefb9de48aea42be6dbd76c61c0 2024-11-20T10:26:42,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/af618ca0948749238f2c6a70a68ed770 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/af618ca0948749238f2c6a70a68ed770 2024-11-20T10:26:42,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/58d37690e484422da0998cd99966bf56 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/58d37690e484422da0998cd99966bf56 2024-11-20T10:26:42,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/66b802cea35a4dc0b96809537198e3da to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/66b802cea35a4dc0b96809537198e3da 2024-11-20T10:26:42,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ea8f7b2320a74a59bc603cad57330239, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a0de538e95e24d6b85429bc1476e9622, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/8c4e0656ac6b4054ae16a2bf42008f9a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/d21ff14dca6f45f1b3a6620a91381373, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/c94aeee543024d9897cd1fe172991efb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/2b1c372e3c3641cb89bf86c522184ea4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/debfe9a9db254503b246ef4530ebf6e5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/22b21ca2b6614eb2906f6b4a7e837099, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/1ac981a0d6c8467c9ee78a5c39fce7d4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a91178a8228d47d28414c74a4a1e09c7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/48477790bee94aefa54c5a53027e723c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/366e95b78cdc46a5adde8bd2683a7fe8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ff4b73fa4a824f3485d5e2ce527cdfc5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bd6f0abb28084573a92d06248a0a5dc8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/2c71f15cdf6e44baa781a561f14a200e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a4b8ec7c778f4be495c85b35ee50328e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/efa3a537424c48dab93a9e57e1644bd5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/8b63aa9d8ec04a62aad525c4d5263565, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/076877ab74b54fcba3413cf594457da7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ee54d4931316447081ff286edb1a9f97, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/b7613cf7c3ca4432bcbbe4a73354abb4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/cb2c68550e1a458c852abced58ad015e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/80ca160bceb64e19a3c2c00f14ba5453, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/54d7ca5a45ab42bf804b3c3abb94e23c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/95cbd2dc028442b894a4feff13924009, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bfd06a5d26184701b18d842e4a7a5307, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bda141f5eeea4881857d8fb4af8f47da, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/3ee199868e3449dda7e1a5791f39b5fb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/4a29c3a11fab4946b75ff8aa217a50f0] to archive 2024-11-20T10:26:42,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:26:42,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ea8f7b2320a74a59bc603cad57330239 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ea8f7b2320a74a59bc603cad57330239 2024-11-20T10:26:42,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a0de538e95e24d6b85429bc1476e9622 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a0de538e95e24d6b85429bc1476e9622 2024-11-20T10:26:42,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/8c4e0656ac6b4054ae16a2bf42008f9a to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/8c4e0656ac6b4054ae16a2bf42008f9a 2024-11-20T10:26:42,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/d21ff14dca6f45f1b3a6620a91381373 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/d21ff14dca6f45f1b3a6620a91381373 2024-11-20T10:26:42,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/c94aeee543024d9897cd1fe172991efb to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/c94aeee543024d9897cd1fe172991efb 2024-11-20T10:26:42,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/2b1c372e3c3641cb89bf86c522184ea4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/2b1c372e3c3641cb89bf86c522184ea4 2024-11-20T10:26:42,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/debfe9a9db254503b246ef4530ebf6e5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/debfe9a9db254503b246ef4530ebf6e5 2024-11-20T10:26:42,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/22b21ca2b6614eb2906f6b4a7e837099 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/22b21ca2b6614eb2906f6b4a7e837099 2024-11-20T10:26:42,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/1ac981a0d6c8467c9ee78a5c39fce7d4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/1ac981a0d6c8467c9ee78a5c39fce7d4 2024-11-20T10:26:42,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a91178a8228d47d28414c74a4a1e09c7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a91178a8228d47d28414c74a4a1e09c7 2024-11-20T10:26:42,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/48477790bee94aefa54c5a53027e723c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/48477790bee94aefa54c5a53027e723c 2024-11-20T10:26:42,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/366e95b78cdc46a5adde8bd2683a7fe8 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/366e95b78cdc46a5adde8bd2683a7fe8 2024-11-20T10:26:42,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ff4b73fa4a824f3485d5e2ce527cdfc5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ff4b73fa4a824f3485d5e2ce527cdfc5 2024-11-20T10:26:42,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bd6f0abb28084573a92d06248a0a5dc8 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bd6f0abb28084573a92d06248a0a5dc8 2024-11-20T10:26:42,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/2c71f15cdf6e44baa781a561f14a200e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/2c71f15cdf6e44baa781a561f14a200e 2024-11-20T10:26:42,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a4b8ec7c778f4be495c85b35ee50328e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a4b8ec7c778f4be495c85b35ee50328e 2024-11-20T10:26:42,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/efa3a537424c48dab93a9e57e1644bd5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/efa3a537424c48dab93a9e57e1644bd5 2024-11-20T10:26:42,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/8b63aa9d8ec04a62aad525c4d5263565 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/8b63aa9d8ec04a62aad525c4d5263565 2024-11-20T10:26:42,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/076877ab74b54fcba3413cf594457da7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/076877ab74b54fcba3413cf594457da7 2024-11-20T10:26:42,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ee54d4931316447081ff286edb1a9f97 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/ee54d4931316447081ff286edb1a9f97 2024-11-20T10:26:42,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/b7613cf7c3ca4432bcbbe4a73354abb4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/b7613cf7c3ca4432bcbbe4a73354abb4 2024-11-20T10:26:42,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/cb2c68550e1a458c852abced58ad015e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/cb2c68550e1a458c852abced58ad015e 2024-11-20T10:26:42,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/80ca160bceb64e19a3c2c00f14ba5453 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/80ca160bceb64e19a3c2c00f14ba5453 2024-11-20T10:26:42,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/54d7ca5a45ab42bf804b3c3abb94e23c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/54d7ca5a45ab42bf804b3c3abb94e23c 2024-11-20T10:26:42,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/95cbd2dc028442b894a4feff13924009 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/95cbd2dc028442b894a4feff13924009 2024-11-20T10:26:42,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bfd06a5d26184701b18d842e4a7a5307 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bfd06a5d26184701b18d842e4a7a5307 2024-11-20T10:26:42,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bda141f5eeea4881857d8fb4af8f47da to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/bda141f5eeea4881857d8fb4af8f47da 2024-11-20T10:26:42,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/3ee199868e3449dda7e1a5791f39b5fb to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/3ee199868e3449dda7e1a5791f39b5fb 2024-11-20T10:26:42,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/4a29c3a11fab4946b75ff8aa217a50f0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/4a29c3a11fab4946b75ff8aa217a50f0 2024-11-20T10:26:42,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/8172da80c83b48f0a6774ae9a19499ce, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/87e06fb8160b4c9084f1bb6f44a132a9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/7014647fe30546ea8df59fd5af9e7a82, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/e2ac80df552c43fc831fb784f9d79a7e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c08ebf9ec48641fd9c8b781122a85947, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/a7f4c9211dce4bb098a863f4bd029698, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9ccefffbbace432f96351cbdf16cec34, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/58397a0ba6ae461cbfdcd1d151df2c1c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/97c7b0b7739e4558bf0e647be4c41c04, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/65efcdeae3de41ec955417be557e63ff, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/1dfb178654b24b709aaebe5d39423740, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9d912396fa0f4e518b78f416dadb678c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/d66c69849b7e4dfa902ac0124aa2f796, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/2651f020aa164a51a364ad725267621e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c1aae30ca13f41f4b59d6a20a7b28ac1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/085298868216414cab12925799c1afbb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/4bf8e234714c41e99b9ad7e097585c64, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/3f2e9e667d6a45c48ac043799a2891ea, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/7e11b76514c54fad8f138661f204e9c2, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/aa6cbc12ac334a41b4cf783fc5c180f9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/19dc9a6f069c4671ab1672227c6854f6, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/b0ef3f635d8a49b7af7f0863169bd227, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/371652e13bd74bfca567aa5363ca233c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/701e6c284ef04673b00a1ecc807102b1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/fe56a1886867430facfadb434eec0ca7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c561714debdf4cc8be7bccb218e1d1c9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/22b64d972d444a5c9b0542d40d3fdd34, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9826f45592ca42eeaff247ea5c2d1fdc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/2073d640ee0f466f85590c5bde4f0566] to archive 2024-11-20T10:26:42,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:26:42,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/8172da80c83b48f0a6774ae9a19499ce to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/8172da80c83b48f0a6774ae9a19499ce 2024-11-20T10:26:42,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/87e06fb8160b4c9084f1bb6f44a132a9 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/87e06fb8160b4c9084f1bb6f44a132a9 2024-11-20T10:26:42,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/7014647fe30546ea8df59fd5af9e7a82 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/7014647fe30546ea8df59fd5af9e7a82 2024-11-20T10:26:42,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/e2ac80df552c43fc831fb784f9d79a7e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/e2ac80df552c43fc831fb784f9d79a7e 2024-11-20T10:26:42,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c08ebf9ec48641fd9c8b781122a85947 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c08ebf9ec48641fd9c8b781122a85947 2024-11-20T10:26:42,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/a7f4c9211dce4bb098a863f4bd029698 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/a7f4c9211dce4bb098a863f4bd029698 2024-11-20T10:26:42,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9ccefffbbace432f96351cbdf16cec34 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9ccefffbbace432f96351cbdf16cec34 2024-11-20T10:26:42,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/58397a0ba6ae461cbfdcd1d151df2c1c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/58397a0ba6ae461cbfdcd1d151df2c1c 2024-11-20T10:26:42,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/97c7b0b7739e4558bf0e647be4c41c04 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/97c7b0b7739e4558bf0e647be4c41c04 2024-11-20T10:26:42,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/65efcdeae3de41ec955417be557e63ff to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/65efcdeae3de41ec955417be557e63ff 2024-11-20T10:26:42,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/1dfb178654b24b709aaebe5d39423740 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/1dfb178654b24b709aaebe5d39423740 2024-11-20T10:26:42,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9d912396fa0f4e518b78f416dadb678c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9d912396fa0f4e518b78f416dadb678c 2024-11-20T10:26:42,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/d66c69849b7e4dfa902ac0124aa2f796 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/d66c69849b7e4dfa902ac0124aa2f796 2024-11-20T10:26:42,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/2651f020aa164a51a364ad725267621e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/2651f020aa164a51a364ad725267621e 2024-11-20T10:26:42,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c1aae30ca13f41f4b59d6a20a7b28ac1 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c1aae30ca13f41f4b59d6a20a7b28ac1 2024-11-20T10:26:42,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/085298868216414cab12925799c1afbb to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/085298868216414cab12925799c1afbb 2024-11-20T10:26:42,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/4bf8e234714c41e99b9ad7e097585c64 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/4bf8e234714c41e99b9ad7e097585c64 2024-11-20T10:26:42,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/3f2e9e667d6a45c48ac043799a2891ea to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/3f2e9e667d6a45c48ac043799a2891ea 2024-11-20T10:26:42,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/7e11b76514c54fad8f138661f204e9c2 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/7e11b76514c54fad8f138661f204e9c2 2024-11-20T10:26:42,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/aa6cbc12ac334a41b4cf783fc5c180f9 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/aa6cbc12ac334a41b4cf783fc5c180f9 2024-11-20T10:26:42,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/19dc9a6f069c4671ab1672227c6854f6 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/19dc9a6f069c4671ab1672227c6854f6 2024-11-20T10:26:42,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/b0ef3f635d8a49b7af7f0863169bd227 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/b0ef3f635d8a49b7af7f0863169bd227 2024-11-20T10:26:42,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/371652e13bd74bfca567aa5363ca233c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/371652e13bd74bfca567aa5363ca233c 2024-11-20T10:26:42,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/701e6c284ef04673b00a1ecc807102b1 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/701e6c284ef04673b00a1ecc807102b1 2024-11-20T10:26:42,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/fe56a1886867430facfadb434eec0ca7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/fe56a1886867430facfadb434eec0ca7 2024-11-20T10:26:42,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c561714debdf4cc8be7bccb218e1d1c9 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/c561714debdf4cc8be7bccb218e1d1c9 2024-11-20T10:26:42,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/22b64d972d444a5c9b0542d40d3fdd34 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/22b64d972d444a5c9b0542d40d3fdd34 2024-11-20T10:26:42,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9826f45592ca42eeaff247ea5c2d1fdc to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/9826f45592ca42eeaff247ea5c2d1fdc 2024-11-20T10:26:42,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/2073d640ee0f466f85590c5bde4f0566 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/2073d640ee0f466f85590c5bde4f0566 2024-11-20T10:26:42,890 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/recovered.edits/448.seqid, newMaxSeqId=448, maxSeqId=1 2024-11-20T10:26:42,890 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. 2024-11-20T10:26:42,890 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for e4730fcd99e6bf6b364225fd77cc4c0d: Waiting for close lock at 1732098401569Running coprocessor pre-close hooks at 1732098401569Disabling compacts and flushes for region at 1732098401569Disabling writes for close at 1732098401569Obtaining lock to block concurrent updates at 1732098401569Preparing flush snapshotting stores in e4730fcd99e6bf6b364225fd77cc4c0d at 1732098401569Finished memstore snapshotting TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732098401569Flushing stores of TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d. at 1732098401570 (+1 ms)Flushing e4730fcd99e6bf6b364225fd77cc4c0d/A: creating writer at 1732098401570Flushing e4730fcd99e6bf6b364225fd77cc4c0d/A: appending metadata at 1732098401572 (+2 ms)Flushing e4730fcd99e6bf6b364225fd77cc4c0d/A: closing flushed file at 1732098401572Flushing e4730fcd99e6bf6b364225fd77cc4c0d/B: creating writer at 1732098401981 (+409 ms)Flushing e4730fcd99e6bf6b364225fd77cc4c0d/B: appending metadata at 1732098401984 (+3 ms)Flushing e4730fcd99e6bf6b364225fd77cc4c0d/B: closing flushed file at 1732098401984Flushing e4730fcd99e6bf6b364225fd77cc4c0d/C: creating writer at 1732098402391 (+407 ms)Flushing e4730fcd99e6bf6b364225fd77cc4c0d/C: appending metadata at 1732098402393 (+2 ms)Flushing e4730fcd99e6bf6b364225fd77cc4c0d/C: closing flushed file at 1732098402393Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d010c62: reopening flushed file at 1732098402800 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b326d6e: reopening flushed file at 1732098402803 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54d6f6f6: reopening flushed file at 1732098402807 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e4730fcd99e6bf6b364225fd77cc4c0d in 1242ms, sequenceid=445, compaction requested=true at 1732098402812 (+5 ms)Writing region close event to WAL at 1732098402887 (+75 ms)Running coprocessor post-close hooks at 1732098402890 (+3 ms)Closed at 1732098402890 2024-11-20T10:26:42,892 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:42,893 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=e4730fcd99e6bf6b364225fd77cc4c0d, regionState=CLOSED 2024-11-20T10:26:42,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:26:42,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-20T10:26:42,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure e4730fcd99e6bf6b364225fd77cc4c0d, server=8c533afd331c,35833,1732098238678 in 1.4780 sec 2024-11-20T10:26:42,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-20T10:26:42,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4730fcd99e6bf6b364225fd77cc4c0d, UNASSIGN in 1.4820 sec 2024-11-20T10:26:42,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-20T10:26:42,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4850 sec 2024-11-20T10:26:42,900 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098402900"}]},"ts":"1732098402900"} 2024-11-20T10:26:42,901 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T10:26:42,901 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T10:26:42,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4930 sec 2024-11-20T10:26:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T10:26:43,557 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:43,558 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T10:26:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:43,559 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T10:26:43,560 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=147, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:43,562 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:43,563 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/recovered.edits] 2024-11-20T10:26:43,565 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/16c7fbc1b3ae4dcb8aad6a6542feb81b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/16c7fbc1b3ae4dcb8aad6a6542feb81b 2024-11-20T10:26:43,566 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/81642b31ad414b9e87d35f21ac8bbcde to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/81642b31ad414b9e87d35f21ac8bbcde 2024-11-20T10:26:43,567 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/dbad60aebec04af68caa716bd5988d02 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/A/dbad60aebec04af68caa716bd5988d02 2024-11-20T10:26:43,568 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/48e9c903b7534dfbab2a37cfbc0ec4bf to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/48e9c903b7534dfbab2a37cfbc0ec4bf 2024-11-20T10:26:43,569 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a4619d546e9a4ddfa135878671b210ea to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/a4619d546e9a4ddfa135878671b210ea 2024-11-20T10:26:43,570 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/b20a245a284b45d2976b890dad006e67 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/B/b20a245a284b45d2976b890dad006e67 2024-11-20T10:26:43,572 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/97fba9d03b0d4c8490289495e257e124 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/97fba9d03b0d4c8490289495e257e124 2024-11-20T10:26:43,573 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/a43cba35da3a47eb9bdec2a3d5df5dc8 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/a43cba35da3a47eb9bdec2a3d5df5dc8 2024-11-20T10:26:43,574 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/d90e31570fef4257b427950eb7183fc3 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/C/d90e31570fef4257b427950eb7183fc3 2024-11-20T10:26:43,576 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/recovered.edits/448.seqid to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d/recovered.edits/448.seqid 2024-11-20T10:26:43,576 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/e4730fcd99e6bf6b364225fd77cc4c0d 2024-11-20T10:26:43,576 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T10:26:43,578 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=147, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:43,579 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T10:26:43,581 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T10:26:43,582 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=147, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:43,582 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T10:26:43,582 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732098403582"}]},"ts":"9223372036854775807"} 2024-11-20T10:26:43,583 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T10:26:43,583 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e4730fcd99e6bf6b364225fd77cc4c0d, NAME => 'TestAcidGuarantees,,1732098370667.e4730fcd99e6bf6b364225fd77cc4c0d.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T10:26:43,583 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T10:26:43,584 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732098403583"}]},"ts":"9223372036854775807"} 2024-11-20T10:26:43,585 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T10:26:43,585 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=147, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:43,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-11-20T10:26:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T10:26:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T10:26:43,667 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:43,674 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=236 (was 239), OpenFileDescriptor=443 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=450 (was 455), ProcessCount=11 (was 11), AvailableMemoryMB=8110 (was 8187) 2024-11-20T10:26:43,681 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=236, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=450, ProcessCount=11, AvailableMemoryMB=8110 2024-11-20T10:26:43,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T10:26:43,682 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T10:26:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:43,684 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T10:26:43,684 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:43,684 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 148 2024-11-20T10:26:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T10:26:43,685 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T10:26:43,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742361_1537 (size=959) 2024-11-20T10:26:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T10:26:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T10:26:44,092 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31 2024-11-20T10:26:44,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742362_1538 (size=53) 2024-11-20T10:26:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T10:26:44,497 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:26:44,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b746a4c7c9036a37fd4050571abf4f3f, disabling compactions & flushes 2024-11-20T10:26:44,498 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:44,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:44,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. after waiting 0 ms 2024-11-20T10:26:44,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:44,498 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:44,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b746a4c7c9036a37fd4050571abf4f3f: Waiting for close lock at 1732098404497Disabling compacts and flushes for region at 1732098404497Disabling writes for close at 1732098404498 (+1 ms)Writing region close event to WAL at 1732098404498Closed at 1732098404498 2024-11-20T10:26:44,499 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T10:26:44,499 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732098404499"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732098404499"}]},"ts":"1732098404499"} 2024-11-20T10:26:44,500 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T10:26:44,501 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T10:26:44,501 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098404501"}]},"ts":"1732098404501"} 2024-11-20T10:26:44,502 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T10:26:44,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b746a4c7c9036a37fd4050571abf4f3f, ASSIGN}] 2024-11-20T10:26:44,503 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b746a4c7c9036a37fd4050571abf4f3f, ASSIGN 2024-11-20T10:26:44,503 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b746a4c7c9036a37fd4050571abf4f3f, ASSIGN; state=OFFLINE, location=8c533afd331c,35833,1732098238678; forceNewPlan=false, retain=false 2024-11-20T10:26:44,654 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=b746a4c7c9036a37fd4050571abf4f3f, regionState=OPENING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:26:44,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b746a4c7c9036a37fd4050571abf4f3f, ASSIGN because future has completed 2024-11-20T10:26:44,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:26:44,810 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:44,810 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7752): Opening region: {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:26:44,810 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:44,810 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:26:44,810 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7794): checking encryption for b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:44,810 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7797): checking classloading for b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:44,811 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:44,812 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b746a4c7c9036a37fd4050571abf4f3f columnFamilyName A 2024-11-20T10:26:44,812 DEBUG [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:44,813 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] regionserver.HStore(327): Store=b746a4c7c9036a37fd4050571abf4f3f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:26:44,813 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:44,814 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b746a4c7c9036a37fd4050571abf4f3f columnFamilyName B 2024-11-20T10:26:44,814 DEBUG [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:44,814 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] regionserver.HStore(327): Store=b746a4c7c9036a37fd4050571abf4f3f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:26:44,814 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:44,815 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b746a4c7c9036a37fd4050571abf4f3f columnFamilyName C 2024-11-20T10:26:44,815 DEBUG [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:44,815 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] regionserver.HStore(327): Store=b746a4c7c9036a37fd4050571abf4f3f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:26:44,815 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1038): replaying wal for b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:44,816 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:44,816 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T10:26:44,817 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1048): stopping wal replay for b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:44,817 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1060): Cleaning up temporary data for b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:44,817 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T10:26:44,818 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1093): writing seq id for b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:44,820 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:26:44,820 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1114): Opened b746a4c7c9036a37fd4050571abf4f3f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60951207, jitterRate=-0.09175623953342438}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T10:26:44,820 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:44,821 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1006): Region open journal for b746a4c7c9036a37fd4050571abf4f3f: Running coprocessor pre-open hook at 1732098404811Writing region info on filesystem at 1732098404811Initializing all the Stores at 1732098404811Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098404811Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098404811Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098404811Cleaning up temporary data from old regions at 1732098404817 (+6 ms)Running coprocessor post-open hooks at 1732098404820 (+3 ms)Region opened successfully at 1732098404821 (+1 ms) 2024-11-20T10:26:44,821 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., pid=150, masterSystemTime=1732098404807 2024-11-20T10:26:44,822 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:44,823 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:44,823 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=b746a4c7c9036a37fd4050571abf4f3f, regionState=OPEN, openSeqNum=2, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:26:44,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:26:44,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-20T10:26:44,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 in 169 msec 2024-11-20T10:26:44,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-20T10:26:44,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b746a4c7c9036a37fd4050571abf4f3f, ASSIGN in 324 msec 2024-11-20T10:26:44,828 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T10:26:44,828 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098404828"}]},"ts":"1732098404828"} 2024-11-20T10:26:44,829 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T10:26:44,829 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T10:26:44,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-20T10:26:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T10:26:45,827 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:45,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f53afef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:45,828 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:45,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:45,829 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6ba830, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:45,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:45,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:45,830 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c1b8df4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:45,831 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:45,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:45,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:45,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:45,833 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T10:26:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 8c533afd331c,33867,1732098237476 2024-11-20T10:26:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@79d4c470 2024-11-20T10:26:45,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T10:26:45,835 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T10:26:45,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T10:26:45,836 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T10:26:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T10:26:45,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742363_1539 (size=995) 2024-11-20T10:26:46,251 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-20T10:26:46,251 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-20T10:26:46,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T10:26:46,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b746a4c7c9036a37fd4050571abf4f3f, REOPEN/MOVE}] 2024-11-20T10:26:46,257 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b746a4c7c9036a37fd4050571abf4f3f, REOPEN/MOVE 2024-11-20T10:26:46,257 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=b746a4c7c9036a37fd4050571abf4f3f, regionState=CLOSING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:26:46,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b746a4c7c9036a37fd4050571abf4f3f, REOPEN/MOVE because future has completed 2024-11-20T10:26:46,258 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T10:26:46,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:26:46,410 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(122): Close b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,410 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T10:26:46,411 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1722): Closing b746a4c7c9036a37fd4050571abf4f3f, disabling compactions & flushes 2024-11-20T10:26:46,411 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:46,411 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:46,411 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. after waiting 0 ms 2024-11-20T10:26:46,411 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:46,414 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T10:26:46,414 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:46,414 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1676): Region close journal for b746a4c7c9036a37fd4050571abf4f3f: Waiting for close lock at 1732098406411Running coprocessor pre-close hooks at 1732098406411Disabling compacts and flushes for region at 1732098406411Disabling writes for close at 1732098406411Writing region close event to WAL at 1732098406411Running coprocessor post-close hooks at 1732098406414 (+3 ms)Closed at 1732098406414 2024-11-20T10:26:46,414 WARN [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionServer(3299): Not adding moved region record: b746a4c7c9036a37fd4050571abf4f3f to self. 2024-11-20T10:26:46,416 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(157): Closed b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,416 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=b746a4c7c9036a37fd4050571abf4f3f, regionState=CLOSED 2024-11-20T10:26:46,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:26:46,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-20T10:26:46,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseRegionProcedure b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 in 159 msec 2024-11-20T10:26:46,420 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b746a4c7c9036a37fd4050571abf4f3f, REOPEN/MOVE; state=CLOSED, location=8c533afd331c,35833,1732098238678; forceNewPlan=false, retain=true 2024-11-20T10:26:46,570 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=b746a4c7c9036a37fd4050571abf4f3f, regionState=OPENING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:26:46,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b746a4c7c9036a37fd4050571abf4f3f, REOPEN/MOVE because future has completed 2024-11-20T10:26:46,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:26:46,726 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:46,726 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7752): Opening region: {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:26:46,726 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,726 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:26:46,726 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7794): checking encryption for b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,726 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7797): checking classloading for b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,727 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,728 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b746a4c7c9036a37fd4050571abf4f3f columnFamilyName A 2024-11-20T10:26:46,729 DEBUG [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:46,729 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] regionserver.HStore(327): Store=b746a4c7c9036a37fd4050571abf4f3f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:26:46,730 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,730 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b746a4c7c9036a37fd4050571abf4f3f columnFamilyName B 2024-11-20T10:26:46,730 DEBUG [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:46,731 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] regionserver.HStore(327): Store=b746a4c7c9036a37fd4050571abf4f3f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:26:46,731 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,731 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b746a4c7c9036a37fd4050571abf4f3f columnFamilyName C 2024-11-20T10:26:46,731 DEBUG [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:46,731 INFO [StoreOpener-b746a4c7c9036a37fd4050571abf4f3f-1 {}] regionserver.HStore(327): Store=b746a4c7c9036a37fd4050571abf4f3f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:26:46,732 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1038): replaying wal for b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,732 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,733 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,733 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1048): stopping wal replay for b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,733 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1060): Cleaning up temporary data for b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,734 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T10:26:46,734 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1093): writing seq id for b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,735 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1114): Opened b746a4c7c9036a37fd4050571abf4f3f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62157314, jitterRate=-0.07378384470939636}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T10:26:46,735 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,735 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1006): Region open journal for b746a4c7c9036a37fd4050571abf4f3f: Running coprocessor pre-open hook at 1732098406726Writing region info on filesystem at 1732098406726Initializing all the Stores at 1732098406727 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098406727Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098406727Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098406727Cleaning up temporary data from old regions at 1732098406733 (+6 ms)Running coprocessor post-open hooks at 1732098406735 (+2 ms)Region opened successfully at 1732098406735 2024-11-20T10:26:46,736 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., pid=155, masterSystemTime=1732098406724 2024-11-20T10:26:46,737 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:46,737 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:46,737 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=b746a4c7c9036a37fd4050571abf4f3f, regionState=OPEN, openSeqNum=5, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:26:46,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:26:46,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=153 2024-11-20T10:26:46,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 in 167 msec 2024-11-20T10:26:46,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-20T10:26:46,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b746a4c7c9036a37fd4050571abf4f3f, REOPEN/MOVE in 484 msec 2024-11-20T10:26:46,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-20T10:26:46,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 489 msec 2024-11-20T10:26:46,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 908 msec 2024-11-20T10:26:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-20T10:26:46,745 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:46,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3071d898, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,746 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:46,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:46,747 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:46,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:46,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:46,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf08a8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:46,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:46,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:46,748 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae6286f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a209286, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:46,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:46,749 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a868dd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:46,750 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@310992cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48fa0b51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:46,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:46,752 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:46,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:46,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:46,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@454ff73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:46,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:46,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:46,753 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70546f62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1130c997, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,753 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:46,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:46,754 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54a0455e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:46,755 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6370035e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29acc1f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:46,756 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:46,756 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582cb80b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:46,757 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:46,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dddcc13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d301717, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:46,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:46,759 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31047e10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:46,760 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bc31bac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36b0a686, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:46,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:46,762 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34a109c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:46,763 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@222b1ec2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18d05b81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:46,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:46,765 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1937ca8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:46,766 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d12cb7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27def64b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:46,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:46,768 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:46,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:46,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:46,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@772262bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:46,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:46,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:46,769 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ad23fb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37c440df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,33867,-1 for getting cluster id 2024-11-20T10:26:46,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:26:46,770 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b99c7a-1668-4a27-a15b-962d0dae8a19' 2024-11-20T10:26:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:26:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b99c7a-1668-4a27-a15b-962d0dae8a19" 2024-11-20T10:26:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b28ae72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,33867,-1] 2024-11-20T10:26:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:26:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:26:46,771 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:26:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6019d9a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:26:46,772 DEBUG [Thread-2430 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:46,772 DEBUG [Thread-2417 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:46,772 DEBUG [Thread-2413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:46,772 DEBUG [Thread-2426 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:46,773 DEBUG [Thread-2424 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:46,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:46,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:46,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:46,773 DEBUG [Thread-2411 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:46,773 DEBUG [Thread-2415 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:46,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:46,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:46,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:46,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:46,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:46,774 DEBUG [Thread-2419 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:46,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:46,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:46,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:46,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:46,775 DEBUG [Thread-2419 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:46,775 DEBUG [Thread-2419 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:46,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:46,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:46,776 DEBUG [Thread-2422 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:46,776 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:46,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:26:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:26:46,777 DEBUG [Thread-2422 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:46,777 DEBUG [Thread-2428 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:26:46,777 DEBUG [Thread-2422 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-20T10:26:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:26:46,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:26:46,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T10:26:46,778 DEBUG [Thread-2428 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,35833,1732098238678, seqNum=-1] 2024-11-20T10:26:46,779 DEBUG [Thread-2428 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:26:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:26:46,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:26:46,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:26:46,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:26:46,781 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:46,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:26:46,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:26:46,782 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:46,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5] 2024-11-20T10:26:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,786 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:46,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:26:46,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c77a4c2674734935a279bd99c56aab70_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098406786/Put/seqid=0 2024-11-20T10:26:46,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098466811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:46,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098466813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:46,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:46,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098466814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:46,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098466815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:46,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:46,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098466815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:46,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:46,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:46,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:46,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742364_1540 (size=12154) 2024-11-20T10:26:46,825 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:46,828 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c77a4c2674734935a279bd99c56aab70_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c77a4c2674734935a279bd99c56aab70_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:46,829 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/bb029c70380a4f87aec9152dc8e7b215, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:46,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/bb029c70380a4f87aec9152dc8e7b215 is 175, key is test_row_0/A:col10/1732098406786/Put/seqid=0 2024-11-20T10:26:46,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742365_1541 (size=30955) 2024-11-20T10:26:46,841 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/bb029c70380a4f87aec9152dc8e7b215 2024-11-20T10:26:46,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/7de5dec6a84f46198c2f1f07c5899424 is 50, key is test_row_0/B:col10/1732098406786/Put/seqid=0 2024-11-20T10:26:46,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742366_1542 (size=12001) 2024-11-20T10:26:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T10:26:46,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098466926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:46,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:46,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098466927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:46,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:46,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:46,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098466927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:46,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098466927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:46,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098466927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:46,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T10:26:46,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:46,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:46,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:46,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:46,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:46,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:46,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:47,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T10:26:47,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:47,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:47,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:47,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:47,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T10:26:47,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098467137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098467137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098467137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098467137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098467146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T10:26:47,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:47,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:47,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:47,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:47,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:47,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:47,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/7de5dec6a84f46198c2f1f07c5899424 2024-11-20T10:26:47,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/632862b8d4f645be9fda522e22184f4c is 50, key is test_row_0/C:col10/1732098406786/Put/seqid=0 2024-11-20T10:26:47,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742367_1543 (size=12001) 2024-11-20T10:26:47,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/632862b8d4f645be9fda522e22184f4c 2024-11-20T10:26:47,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/bb029c70380a4f87aec9152dc8e7b215 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/bb029c70380a4f87aec9152dc8e7b215 2024-11-20T10:26:47,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/bb029c70380a4f87aec9152dc8e7b215, entries=150, sequenceid=17, filesize=30.2 K 2024-11-20T10:26:47,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/7de5dec6a84f46198c2f1f07c5899424 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/7de5dec6a84f46198c2f1f07c5899424 2024-11-20T10:26:47,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/7de5dec6a84f46198c2f1f07c5899424, entries=150, sequenceid=17, filesize=11.7 K 2024-11-20T10:26:47,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/632862b8d4f645be9fda522e22184f4c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/632862b8d4f645be9fda522e22184f4c 2024-11-20T10:26:47,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/632862b8d4f645be9fda522e22184f4c, entries=150, sequenceid=17, filesize=11.7 K 2024-11-20T10:26:47,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for b746a4c7c9036a37fd4050571abf4f3f in 538ms, sequenceid=17, compaction requested=false 2024-11-20T10:26:47,324 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T10:26:47,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:26:47,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T10:26:47,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:47,395 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T10:26:47,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204259cb24d12e42289f33a16725cf8f3a_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098406814/Put/seqid=0 2024-11-20T10:26:47,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742368_1544 (size=12154) 2024-11-20T10:26:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T10:26:47,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:47,410 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204259cb24d12e42289f33a16725cf8f3a_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204259cb24d12e42289f33a16725cf8f3a_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:47,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/c51eab12846748d79cbef1f8e7212c5f, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:47,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/c51eab12846748d79cbef1f8e7212c5f is 175, key is test_row_0/A:col10/1732098406814/Put/seqid=0 2024-11-20T10:26:47,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742369_1545 (size=30955) 2024-11-20T10:26:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:47,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:47,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098467453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098467454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098467454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098467455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098467456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098467556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098467566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098467566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098467566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098467767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098467776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098467777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098467777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:47,816 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/c51eab12846748d79cbef1f8e7212c5f 2024-11-20T10:26:47,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/455488e4bf4e4bcfa906d1daadb2b89b is 50, key is test_row_0/B:col10/1732098406814/Put/seqid=0 2024-11-20T10:26:47,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742370_1546 (size=12001) 2024-11-20T10:26:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T10:26:47,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098467966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:48,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:48,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098468077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:48,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098468086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098468086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:48,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098468086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:48,226 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/455488e4bf4e4bcfa906d1daadb2b89b 2024-11-20T10:26:48,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/deec465d31844c3cbc3c1426c8570216 is 50, key is test_row_0/C:col10/1732098406814/Put/seqid=0 2024-11-20T10:26:48,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742371_1547 (size=12001) 2024-11-20T10:26:48,409 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T10:26:48,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098468586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:48,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098468596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098468596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098468596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:48,635 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/deec465d31844c3cbc3c1426c8570216 2024-11-20T10:26:48,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/c51eab12846748d79cbef1f8e7212c5f as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/c51eab12846748d79cbef1f8e7212c5f 2024-11-20T10:26:48,646 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/c51eab12846748d79cbef1f8e7212c5f, entries=150, sequenceid=41, filesize=30.2 K 2024-11-20T10:26:48,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/455488e4bf4e4bcfa906d1daadb2b89b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/455488e4bf4e4bcfa906d1daadb2b89b 2024-11-20T10:26:48,650 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/455488e4bf4e4bcfa906d1daadb2b89b, entries=150, sequenceid=41, filesize=11.7 K 2024-11-20T10:26:48,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/deec465d31844c3cbc3c1426c8570216 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/deec465d31844c3cbc3c1426c8570216 2024-11-20T10:26:48,655 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/deec465d31844c3cbc3c1426c8570216, entries=150, sequenceid=41, filesize=11.7 K 2024-11-20T10:26:48,655 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b746a4c7c9036a37fd4050571abf4f3f in 1260ms, sequenceid=41, compaction requested=false 2024-11-20T10:26:48,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:26:48,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:48,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-20T10:26:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-20T10:26:48,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-20T10:26:48,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8760 sec 2024-11-20T10:26:48,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8840 sec 2024-11-20T10:26:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T10:26:48,927 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:48,927 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-20T10:26:48,929 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T10:26:48,930 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:48,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:48,988 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:48,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T10:26:48,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120035901fd36f5489ba7080c93e6ec9c45_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098407450/Put/seqid=0 2024-11-20T10:26:49,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742372_1548 (size=14594) 2024-11-20T10:26:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T10:26:49,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098469074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:49,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T10:26:49,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:49,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:49,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:49,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:49,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098469187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:49,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:49,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T10:26:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:49,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T10:26:49,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T10:26:49,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:49,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:49,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:49,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:49,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098469397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:49,413 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:49,416 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120035901fd36f5489ba7080c93e6ec9c45_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120035901fd36f5489ba7080c93e6ec9c45_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:49,416 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/263865b9920a4000b10c4a76ce8836aa, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:49,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/263865b9920a4000b10c4a76ce8836aa is 175, key is test_row_0/A:col10/1732098407450/Put/seqid=0 2024-11-20T10:26:49,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742373_1549 (size=39549) 2024-11-20T10:26:49,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T10:26:49,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:49,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:49,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:49,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T10:26:49,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098469597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:49,599 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in b746a4c7c9036a37fd4050571abf4f3f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2146 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:49,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098469606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:49,609 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_1' in b746a4c7c9036a37fd4050571abf4f3f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:49,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:49,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098469616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:49,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098469616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:49,618 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in b746a4c7c9036a37fd4050571abf4f3f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:49,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:49,619 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in b746a4c7c9036a37fd4050571abf4f3f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:49,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:49,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T10:26:49,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:49,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:49,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:49,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098469706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:49,820 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/263865b9920a4000b10c4a76ce8836aa 2024-11-20T10:26:49,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/34568215e7f240458ba1f20eb35ca7cf is 50, key is test_row_0/B:col10/1732098407450/Put/seqid=0 2024-11-20T10:26:49,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742374_1550 (size=12001) 2024-11-20T10:26:49,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T10:26:49,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:49,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:49,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:49,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T10:26:49,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:49,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:49,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:49,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T10:26:50,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T10:26:50,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:50,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:50,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:50,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:50,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:50,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:50,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098470216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:50,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/34568215e7f240458ba1f20eb35ca7cf 2024-11-20T10:26:50,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/344870965bae4115a56439c71f8f772d is 50, key is test_row_0/C:col10/1732098407450/Put/seqid=0 2024-11-20T10:26:50,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742375_1551 (size=12001) 2024-11-20T10:26:50,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T10:26:50,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:50,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:50,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:50,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:50,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:50,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:50,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T10:26:50,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:50,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:50,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:50,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:50,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:50,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:50,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T10:26:50,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:50,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:50,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:50,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:50,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:50,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/344870965bae4115a56439c71f8f772d 2024-11-20T10:26:50,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/263865b9920a4000b10c4a76ce8836aa as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/263865b9920a4000b10c4a76ce8836aa 2024-11-20T10:26:50,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/263865b9920a4000b10c4a76ce8836aa, entries=200, sequenceid=54, filesize=38.6 K 2024-11-20T10:26:50,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/34568215e7f240458ba1f20eb35ca7cf as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/34568215e7f240458ba1f20eb35ca7cf 2024-11-20T10:26:50,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/34568215e7f240458ba1f20eb35ca7cf, entries=150, sequenceid=54, filesize=11.7 K 2024-11-20T10:26:50,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/344870965bae4115a56439c71f8f772d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/344870965bae4115a56439c71f8f772d 2024-11-20T10:26:50,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/344870965bae4115a56439c71f8f772d, entries=150, sequenceid=54, filesize=11.7 K 2024-11-20T10:26:50,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b746a4c7c9036a37fd4050571abf4f3f in 1675ms, sequenceid=54, compaction requested=true 2024-11-20T10:26:50,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:26:50,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b746a4c7c9036a37fd4050571abf4f3f:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:26:50,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:50,663 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:50,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b746a4c7c9036a37fd4050571abf4f3f:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:26:50,663 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:50,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:50,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b746a4c7c9036a37fd4050571abf4f3f:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:26:50,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:50,664 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:50,664 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:50,664 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): b746a4c7c9036a37fd4050571abf4f3f/B is initiating minor compaction (all files) 2024-11-20T10:26:50,664 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b746a4c7c9036a37fd4050571abf4f3f/B in TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:50,664 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/7de5dec6a84f46198c2f1f07c5899424, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/455488e4bf4e4bcfa906d1daadb2b89b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/34568215e7f240458ba1f20eb35ca7cf] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp, totalSize=35.2 K 2024-11-20T10:26:50,664 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): b746a4c7c9036a37fd4050571abf4f3f/A is initiating minor compaction (all files) 2024-11-20T10:26:50,664 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b746a4c7c9036a37fd4050571abf4f3f/A in TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:50,664 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/bb029c70380a4f87aec9152dc8e7b215, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/c51eab12846748d79cbef1f8e7212c5f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/263865b9920a4000b10c4a76ce8836aa] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp, totalSize=99.1 K 2024-11-20T10:26:50,664 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 7de5dec6a84f46198c2f1f07c5899424, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732098406785 2024-11-20T10:26:50,664 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:50,665 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/bb029c70380a4f87aec9152dc8e7b215, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/c51eab12846748d79cbef1f8e7212c5f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/263865b9920a4000b10c4a76ce8836aa] 2024-11-20T10:26:50,665 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 455488e4bf4e4bcfa906d1daadb2b89b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732098406811 2024-11-20T10:26:50,665 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb029c70380a4f87aec9152dc8e7b215, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732098406785 2024-11-20T10:26:50,665 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 34568215e7f240458ba1f20eb35ca7cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732098407450 2024-11-20T10:26:50,665 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c51eab12846748d79cbef1f8e7212c5f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732098406811 2024-11-20T10:26:50,666 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 263865b9920a4000b10c4a76ce8836aa, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732098407450 2024-11-20T10:26:50,672 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b746a4c7c9036a37fd4050571abf4f3f#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:50,673 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/39ece95df412406a81ad5ca23e0bcfc8 is 50, key is test_row_0/B:col10/1732098407450/Put/seqid=0 2024-11-20T10:26:50,681 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:50,683 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112019207401feeb4e5ba3e6a45941ad2d36_b746a4c7c9036a37fd4050571abf4f3f store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:50,684 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112019207401feeb4e5ba3e6a45941ad2d36_b746a4c7c9036a37fd4050571abf4f3f, store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:50,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742376_1552 (size=12104) 2024-11-20T10:26:50,685 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112019207401feeb4e5ba3e6a45941ad2d36_b746a4c7c9036a37fd4050571abf4f3f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:50,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742377_1553 (size=4469) 2024-11-20T10:26:50,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T10:26:50,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:50,761 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T10:26:50,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120280ac3587da4435e822f4ccb8c2632e4_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098409068/Put/seqid=0 2024-11-20T10:26:50,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742378_1554 (size=12154) 2024-11-20T10:26:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T10:26:51,097 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b746a4c7c9036a37fd4050571abf4f3f#A#compaction#469 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:51,097 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/2503df1e57ba48939e2074f35b14eda7 is 175, key is test_row_0/A:col10/1732098407450/Put/seqid=0 2024-11-20T10:26:51,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742379_1555 (size=31058) 2024-11-20T10:26:51,101 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/39ece95df412406a81ad5ca23e0bcfc8 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/39ece95df412406a81ad5ca23e0bcfc8 2024-11-20T10:26:51,106 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b746a4c7c9036a37fd4050571abf4f3f/B of b746a4c7c9036a37fd4050571abf4f3f into 39ece95df412406a81ad5ca23e0bcfc8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:51,106 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:26:51,106 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., storeName=b746a4c7c9036a37fd4050571abf4f3f/B, priority=13, startTime=1732098410663; duration=0sec 2024-11-20T10:26:51,106 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:51,106 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b746a4c7c9036a37fd4050571abf4f3f:B 2024-11-20T10:26:51,107 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:26:51,107 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/2503df1e57ba48939e2074f35b14eda7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/2503df1e57ba48939e2074f35b14eda7 2024-11-20T10:26:51,107 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:26:51,108 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): b746a4c7c9036a37fd4050571abf4f3f/C is initiating minor compaction (all files) 2024-11-20T10:26:51,108 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b746a4c7c9036a37fd4050571abf4f3f/C in TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:51,108 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/632862b8d4f645be9fda522e22184f4c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/deec465d31844c3cbc3c1426c8570216, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/344870965bae4115a56439c71f8f772d] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp, totalSize=35.2 K 2024-11-20T10:26:51,108 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 632862b8d4f645be9fda522e22184f4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732098406785 2024-11-20T10:26:51,109 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting deec465d31844c3cbc3c1426c8570216, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732098406811 2024-11-20T10:26:51,109 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 344870965bae4115a56439c71f8f772d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732098407450 2024-11-20T10:26:51,113 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b746a4c7c9036a37fd4050571abf4f3f/A of b746a4c7c9036a37fd4050571abf4f3f into 2503df1e57ba48939e2074f35b14eda7(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:51,114 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:26:51,114 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., storeName=b746a4c7c9036a37fd4050571abf4f3f/A, priority=13, startTime=1732098410663; duration=0sec 2024-11-20T10:26:51,114 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:51,114 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b746a4c7c9036a37fd4050571abf4f3f:A 2024-11-20T10:26:51,116 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b746a4c7c9036a37fd4050571abf4f3f#C#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:51,117 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/056c2dbcdb6a4253960feccd74cea015 is 50, key is test_row_0/C:col10/1732098407450/Put/seqid=0 2024-11-20T10:26:51,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742380_1556 (size=12104) 2024-11-20T10:26:51,134 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/056c2dbcdb6a4253960feccd74cea015 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/056c2dbcdb6a4253960feccd74cea015 2024-11-20T10:26:51,137 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b746a4c7c9036a37fd4050571abf4f3f/C of b746a4c7c9036a37fd4050571abf4f3f into 056c2dbcdb6a4253960feccd74cea015(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:51,137 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:26:51,137 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., storeName=b746a4c7c9036a37fd4050571abf4f3f/C, priority=13, startTime=1732098410663; duration=0sec 2024-11-20T10:26:51,137 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:51,137 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b746a4c7c9036a37fd4050571abf4f3f:C 2024-11-20T10:26:51,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:51,175 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120280ac3587da4435e822f4ccb8c2632e4_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120280ac3587da4435e822f4ccb8c2632e4_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:51,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/eee1474081ae45f192fe80239e0fdfd9, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:51,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/eee1474081ae45f192fe80239e0fdfd9 is 175, key is test_row_0/A:col10/1732098409068/Put/seqid=0 2024-11-20T10:26:51,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742381_1557 (size=30955) 2024-11-20T10:26:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:51,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:51,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098471265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:51,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098471376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:51,579 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/eee1474081ae45f192fe80239e0fdfd9 2024-11-20T10:26:51,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/aea684b389014ce7940c983fc6187d46 is 50, key is test_row_0/B:col10/1732098409068/Put/seqid=0 2024-11-20T10:26:51,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742382_1558 (size=12001) 2024-11-20T10:26:51,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098471587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:51,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098471616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:51,618 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in b746a4c7c9036a37fd4050571abf4f3f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:51,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098471637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:51,639 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_1' in b746a4c7c9036a37fd4050571abf4f3f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:51,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:51,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098471647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:51,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098471647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:51,650 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in b746a4c7c9036a37fd4050571abf4f3f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:51,650 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in b746a4c7c9036a37fd4050571abf4f3f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:51,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098471897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:51,990 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/aea684b389014ce7940c983fc6187d46 2024-11-20T10:26:51,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/60d5c6a59aaa4c69ab51638fa37067eb is 50, key is test_row_0/C:col10/1732098409068/Put/seqid=0 2024-11-20T10:26:52,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742383_1559 (size=12001) 2024-11-20T10:26:52,402 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/60d5c6a59aaa4c69ab51638fa37067eb 2024-11-20T10:26:52,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/eee1474081ae45f192fe80239e0fdfd9 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/eee1474081ae45f192fe80239e0fdfd9 2024-11-20T10:26:52,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:52,408 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/eee1474081ae45f192fe80239e0fdfd9, entries=150, sequenceid=77, filesize=30.2 K 2024-11-20T10:26:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098472407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:52,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/aea684b389014ce7940c983fc6187d46 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/aea684b389014ce7940c983fc6187d46 2024-11-20T10:26:52,412 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/aea684b389014ce7940c983fc6187d46, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T10:26:52,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/60d5c6a59aaa4c69ab51638fa37067eb as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/60d5c6a59aaa4c69ab51638fa37067eb 2024-11-20T10:26:52,415 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/60d5c6a59aaa4c69ab51638fa37067eb, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T10:26:52,416 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for b746a4c7c9036a37fd4050571abf4f3f in 1655ms, sequenceid=77, compaction requested=false 2024-11-20T10:26:52,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:26:52,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:52,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-20T10:26:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-20T10:26:52,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-20T10:26:52,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4880 sec 2024-11-20T10:26:52,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 3.4930 sec 2024-11-20T10:26:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T10:26:53,107 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:53,107 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-20T10:26:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T10:26:53,109 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:53,109 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:53,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T10:26:53,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T10:26:53,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:53,261 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T10:26:53,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120171a5540c328461a926bee1aa14f2830_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098411255/Put/seqid=0 2024-11-20T10:26:53,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742384_1560 (size=12154) 2024-11-20T10:26:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:53,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:53,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T10:26:53,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:53,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098473493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:53,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:53,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098473607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:53,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:53,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:53,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:53,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:53,676 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120171a5540c328461a926bee1aa14f2830_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120171a5540c328461a926bee1aa14f2830_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:53,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/7add99ae38a0412b920964bc2f3c8249, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:53,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/7add99ae38a0412b920964bc2f3c8249 is 175, key is test_row_0/A:col10/1732098411255/Put/seqid=0 2024-11-20T10:26:53,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742385_1561 (size=30955) 2024-11-20T10:26:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T10:26:53,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098473816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:54,089 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/7add99ae38a0412b920964bc2f3c8249 2024-11-20T10:26:54,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/7544e4aff07c496980d55d61e6912c61 is 50, key is test_row_0/B:col10/1732098411255/Put/seqid=0 2024-11-20T10:26:54,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742386_1562 (size=12001) 2024-11-20T10:26:54,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098474126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:54,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:54,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:54,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T10:26:54,501 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/7544e4aff07c496980d55d61e6912c61 2024-11-20T10:26:54,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/e9cbbd86c4bc422685c9399a375b00ad is 50, key is test_row_0/C:col10/1732098411255/Put/seqid=0 2024-11-20T10:26:54,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742387_1563 (size=12001) 2024-11-20T10:26:54,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098474636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:54,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:54,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:54,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:54,965 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/e9cbbd86c4bc422685c9399a375b00ad 2024-11-20T10:26:54,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/7add99ae38a0412b920964bc2f3c8249 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/7add99ae38a0412b920964bc2f3c8249 2024-11-20T10:26:54,973 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/7add99ae38a0412b920964bc2f3c8249, entries=150, sequenceid=93, filesize=30.2 K 2024-11-20T10:26:54,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/7544e4aff07c496980d55d61e6912c61 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/7544e4aff07c496980d55d61e6912c61 2024-11-20T10:26:54,977 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/7544e4aff07c496980d55d61e6912c61, entries=150, sequenceid=93, filesize=11.7 K 2024-11-20T10:26:54,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/e9cbbd86c4bc422685c9399a375b00ad as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/e9cbbd86c4bc422685c9399a375b00ad 2024-11-20T10:26:54,980 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/e9cbbd86c4bc422685c9399a375b00ad, entries=150, sequenceid=93, filesize=11.7 K 2024-11-20T10:26:54,981 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b746a4c7c9036a37fd4050571abf4f3f in 1720ms, sequenceid=93, compaction requested=true 2024-11-20T10:26:54,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:26:54,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:54,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-20T10:26:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-20T10:26:54,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-20T10:26:54,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8730 sec 2024-11-20T10:26:54,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8770 sec 2024-11-20T10:26:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T10:26:55,257 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:55,257 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-20T10:26:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T10:26:55,259 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:55,259 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:55,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T10:26:55,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T10:26:55,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:55,411 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T10:26:55,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206d773b0e0c8247b6ae0e2ac05f66d96f_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098413476/Put/seqid=0 2024-11-20T10:26:55,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742388_1564 (size=12154) 2024-11-20T10:26:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T10:26:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:55,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:55,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098475669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:55,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098475669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:55,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098475670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:55,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098475686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:55,689 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_1' in b746a4c7c9036a37fd4050571abf4f3f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:55,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098475696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:55,700 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in b746a4c7c9036a37fd4050571abf4f3f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:26:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:55,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098475776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:55,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098475776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098475776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:55,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:55,822 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206d773b0e0c8247b6ae0e2ac05f66d96f_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206d773b0e0c8247b6ae0e2ac05f66d96f_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:55,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/9bddc6219ad2474db519b12f5bd4eda4, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:55,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/9bddc6219ad2474db519b12f5bd4eda4 is 175, key is test_row_0/A:col10/1732098413476/Put/seqid=0 2024-11-20T10:26:55,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742389_1565 (size=30955) 2024-11-20T10:26:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T10:26:55,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098475986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:55,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098475986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098475986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:56,226 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/9bddc6219ad2474db519b12f5bd4eda4 2024-11-20T10:26:56,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/486eb2c38eb24fdc9cb90262dc3efda6 is 50, key is test_row_0/B:col10/1732098413476/Put/seqid=0 2024-11-20T10:26:56,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742390_1566 (size=12001) 2024-11-20T10:26:56,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098476296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:56,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098476296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:56,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098476296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T10:26:56,535 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T10:26:56,640 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/486eb2c38eb24fdc9cb90262dc3efda6 2024-11-20T10:26:56,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/28e54bc43adf4418b548ec85fa827379 is 50, key is test_row_0/C:col10/1732098413476/Put/seqid=0 2024-11-20T10:26:56,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742391_1567 (size=12001) 2024-11-20T10:26:56,649 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/28e54bc43adf4418b548ec85fa827379 2024-11-20T10:26:56,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/9bddc6219ad2474db519b12f5bd4eda4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/9bddc6219ad2474db519b12f5bd4eda4 2024-11-20T10:26:56,657 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/9bddc6219ad2474db519b12f5bd4eda4, entries=150, sequenceid=116, filesize=30.2 K 2024-11-20T10:26:56,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/486eb2c38eb24fdc9cb90262dc3efda6 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/486eb2c38eb24fdc9cb90262dc3efda6 2024-11-20T10:26:56,663 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/486eb2c38eb24fdc9cb90262dc3efda6, entries=150, sequenceid=116, filesize=11.7 K 2024-11-20T10:26:56,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/28e54bc43adf4418b548ec85fa827379 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/28e54bc43adf4418b548ec85fa827379 2024-11-20T10:26:56,667 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/28e54bc43adf4418b548ec85fa827379, entries=150, sequenceid=116, filesize=11.7 K 2024-11-20T10:26:56,668 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for b746a4c7c9036a37fd4050571abf4f3f in 1257ms, sequenceid=116, compaction requested=true 2024-11-20T10:26:56,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:26:56,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:56,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-20T10:26:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-20T10:26:56,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-20T10:26:56,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4100 sec 2024-11-20T10:26:56,673 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.4150 sec 2024-11-20T10:26:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:56,809 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:56,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-20T10:26:56,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b06054c007d44866b2a37f62aff4a154_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098415668/Put/seqid=0 2024-11-20T10:26:56,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742392_1568 (size=14644) 2024-11-20T10:26:56,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098476889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:56,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098476889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:56,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098476892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:56,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098476997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:56,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:56,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:56,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:57,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098477007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:57,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098477007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:57,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:57,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:57,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098477207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:57,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:57,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098477217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:57,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098477217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:57,233 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:57,236 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b06054c007d44866b2a37f62aff4a154_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b06054c007d44866b2a37f62aff4a154_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:57,236 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/b3cf24387bd8492cb670c0b841758f0e, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:57,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/b3cf24387bd8492cb670c0b841758f0e is 175, key is test_row_0/A:col10/1732098415668/Put/seqid=0 2024-11-20T10:26:57,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742393_1569 (size=39599) 2024-11-20T10:26:57,240 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/b3cf24387bd8492cb670c0b841758f0e 2024-11-20T10:26:57,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/268bed34008f42e6b87298e09bb6b04a is 50, key is test_row_0/B:col10/1732098415668/Put/seqid=0 2024-11-20T10:26:57,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742394_1570 (size=12051) 2024-11-20T10:26:57,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/268bed34008f42e6b87298e09bb6b04a 2024-11-20T10:26:57,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/06dc1be11ece4ab4878b09ef3b8824a5 is 50, key is test_row_0/C:col10/1732098415668/Put/seqid=0 2024-11-20T10:26:57,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742395_1571 (size=12051) 2024-11-20T10:26:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T10:26:57,407 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:57,407 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-20T10:26:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T10:26:57,412 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:57,413 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:57,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T10:26:57,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098477516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:57,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098477527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:57,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:57,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098477527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:57,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-20T10:26:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:57,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:57,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/06dc1be11ece4ab4878b09ef3b8824a5 2024-11-20T10:26:57,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/b3cf24387bd8492cb670c0b841758f0e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/b3cf24387bd8492cb670c0b841758f0e 2024-11-20T10:26:57,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/b3cf24387bd8492cb670c0b841758f0e, entries=200, sequenceid=130, filesize=38.7 K 2024-11-20T10:26:57,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/268bed34008f42e6b87298e09bb6b04a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/268bed34008f42e6b87298e09bb6b04a 2024-11-20T10:26:57,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/268bed34008f42e6b87298e09bb6b04a, entries=150, sequenceid=130, filesize=11.8 K 2024-11-20T10:26:57,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/06dc1be11ece4ab4878b09ef3b8824a5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/06dc1be11ece4ab4878b09ef3b8824a5 2024-11-20T10:26:57,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/06dc1be11ece4ab4878b09ef3b8824a5, entries=150, sequenceid=130, filesize=11.8 K 2024-11-20T10:26:57,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for b746a4c7c9036a37fd4050571abf4f3f in 865ms, sequenceid=130, compaction requested=true 2024-11-20T10:26:57,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:26:57,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b746a4c7c9036a37fd4050571abf4f3f:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:26:57,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:57,675 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T10:26:57,675 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T10:26:57,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b746a4c7c9036a37fd4050571abf4f3f:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:26:57,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:57,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b746a4c7c9036a37fd4050571abf4f3f:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:26:57,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:57,676 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 163522 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T10:26:57,676 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): b746a4c7c9036a37fd4050571abf4f3f/A is initiating minor compaction (all files) 2024-11-20T10:26:57,676 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b746a4c7c9036a37fd4050571abf4f3f/A in TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:57,676 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/2503df1e57ba48939e2074f35b14eda7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/eee1474081ae45f192fe80239e0fdfd9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/7add99ae38a0412b920964bc2f3c8249, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/9bddc6219ad2474db519b12f5bd4eda4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/b3cf24387bd8492cb670c0b841758f0e] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp, totalSize=159.7 K 2024-11-20T10:26:57,676 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:57,676 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/2503df1e57ba48939e2074f35b14eda7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/eee1474081ae45f192fe80239e0fdfd9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/7add99ae38a0412b920964bc2f3c8249, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/9bddc6219ad2474db519b12f5bd4eda4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/b3cf24387bd8492cb670c0b841758f0e] 2024-11-20T10:26:57,677 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T10:26:57,677 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2503df1e57ba48939e2074f35b14eda7, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732098407450 2024-11-20T10:26:57,677 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): b746a4c7c9036a37fd4050571abf4f3f/B is initiating minor compaction (all files) 2024-11-20T10:26:57,677 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b746a4c7c9036a37fd4050571abf4f3f/B in TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:57,677 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/39ece95df412406a81ad5ca23e0bcfc8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/aea684b389014ce7940c983fc6187d46, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/7544e4aff07c496980d55d61e6912c61, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/486eb2c38eb24fdc9cb90262dc3efda6, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/268bed34008f42e6b87298e09bb6b04a] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp, totalSize=58.7 K 2024-11-20T10:26:57,677 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting eee1474081ae45f192fe80239e0fdfd9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732098409064 2024-11-20T10:26:57,678 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 39ece95df412406a81ad5ca23e0bcfc8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732098407450 2024-11-20T10:26:57,678 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7add99ae38a0412b920964bc2f3c8249, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732098411247 2024-11-20T10:26:57,678 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting aea684b389014ce7940c983fc6187d46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732098409064 2024-11-20T10:26:57,678 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bddc6219ad2474db519b12f5bd4eda4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732098413476 2024-11-20T10:26:57,678 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 7544e4aff07c496980d55d61e6912c61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732098411247 2024-11-20T10:26:57,678 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3cf24387bd8492cb670c0b841758f0e, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732098415664 2024-11-20T10:26:57,678 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 486eb2c38eb24fdc9cb90262dc3efda6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732098413476 2024-11-20T10:26:57,679 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 268bed34008f42e6b87298e09bb6b04a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732098415664 2024-11-20T10:26:57,695 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b746a4c7c9036a37fd4050571abf4f3f#B#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:57,695 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/c1ce3da6a17441949dc8ecc971621a43 is 50, key is test_row_0/B:col10/1732098415668/Put/seqid=0 2024-11-20T10:26:57,696 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:57,701 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120e60cb392f6164ff4af8f71215671498b_b746a4c7c9036a37fd4050571abf4f3f store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:57,704 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120e60cb392f6164ff4af8f71215671498b_b746a4c7c9036a37fd4050571abf4f3f, store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:57,704 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e60cb392f6164ff4af8f71215671498b_b746a4c7c9036a37fd4050571abf4f3f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:57,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-20T10:26:57,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:57,718 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-20T10:26:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T10:26:57,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742397_1573 (size=4469) 2024-11-20T10:26:57,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742396_1572 (size=12325) 2024-11-20T10:26:57,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207b5fb9e2e2134bf885edd91493564d82_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098416890/Put/seqid=0 2024-11-20T10:26:57,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742398_1574 (size=12304) 2024-11-20T10:26:58,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:58,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T10:26:58,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:58,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098478054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098478055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:58,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098478056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:58,138 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b746a4c7c9036a37fd4050571abf4f3f#A#compaction#484 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:58,139 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/6887046602d04cc29dc231a934f487f4 is 175, key is test_row_0/A:col10/1732098415668/Put/seqid=0 2024-11-20T10:26:58,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742399_1575 (size=31279) 2024-11-20T10:26:58,152 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/6887046602d04cc29dc231a934f487f4 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/6887046602d04cc29dc231a934f487f4 2024-11-20T10:26:58,152 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/c1ce3da6a17441949dc8ecc971621a43 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/c1ce3da6a17441949dc8ecc971621a43 2024-11-20T10:26:58,155 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b746a4c7c9036a37fd4050571abf4f3f/B of b746a4c7c9036a37fd4050571abf4f3f into c1ce3da6a17441949dc8ecc971621a43(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:58,156 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:26:58,156 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., storeName=b746a4c7c9036a37fd4050571abf4f3f/B, priority=11, startTime=1732098417675; duration=0sec 2024-11-20T10:26:58,156 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:26:58,156 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b746a4c7c9036a37fd4050571abf4f3f:B 2024-11-20T10:26:58,156 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b746a4c7c9036a37fd4050571abf4f3f/A of b746a4c7c9036a37fd4050571abf4f3f into 6887046602d04cc29dc231a934f487f4(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:58,156 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:26:58,156 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T10:26:58,156 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., storeName=b746a4c7c9036a37fd4050571abf4f3f/A, priority=11, startTime=1732098417675; duration=0sec 2024-11-20T10:26:58,156 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:58,156 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b746a4c7c9036a37fd4050571abf4f3f:A 2024-11-20T10:26:58,157 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T10:26:58,157 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): b746a4c7c9036a37fd4050571abf4f3f/C is initiating minor compaction (all files) 2024-11-20T10:26:58,157 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b746a4c7c9036a37fd4050571abf4f3f/C in TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:58,157 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/056c2dbcdb6a4253960feccd74cea015, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/60d5c6a59aaa4c69ab51638fa37067eb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/e9cbbd86c4bc422685c9399a375b00ad, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/28e54bc43adf4418b548ec85fa827379, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/06dc1be11ece4ab4878b09ef3b8824a5] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp, totalSize=58.7 K 2024-11-20T10:26:58,157 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 056c2dbcdb6a4253960feccd74cea015, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732098407450 2024-11-20T10:26:58,158 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 60d5c6a59aaa4c69ab51638fa37067eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732098409064 2024-11-20T10:26:58,158 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting e9cbbd86c4bc422685c9399a375b00ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732098411247 2024-11-20T10:26:58,158 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 28e54bc43adf4418b548ec85fa827379, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732098413476 2024-11-20T10:26:58,158 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 06dc1be11ece4ab4878b09ef3b8824a5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732098415664 2024-11-20T10:26:58,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:58,161 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207b5fb9e2e2134bf885edd91493564d82_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207b5fb9e2e2134bf885edd91493564d82_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/e798654fd20b47ae98617a27245e7f01, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/e798654fd20b47ae98617a27245e7f01 is 175, key is test_row_0/A:col10/1732098416890/Put/seqid=0 2024-11-20T10:26:58,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742400_1576 (size=31105) 2024-11-20T10:26:58,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:58,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098478166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:58,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:58,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098478167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:58,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:58,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098478167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:58,176 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b746a4c7c9036a37fd4050571abf4f3f#C#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:26:58,177 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/4b941a2fa7e14d5b8f982ca9176897f0 is 50, key is test_row_0/C:col10/1732098415668/Put/seqid=0 2024-11-20T10:26:58,177 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/e798654fd20b47ae98617a27245e7f01 2024-11-20T10:26:58,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742401_1577 (size=12325) 2024-11-20T10:26:58,187 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/4b941a2fa7e14d5b8f982ca9176897f0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/4b941a2fa7e14d5b8f982ca9176897f0 2024-11-20T10:26:58,191 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b746a4c7c9036a37fd4050571abf4f3f/C of b746a4c7c9036a37fd4050571abf4f3f into 4b941a2fa7e14d5b8f982ca9176897f0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:26:58,191 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:26:58,191 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., storeName=b746a4c7c9036a37fd4050571abf4f3f/C, priority=11, startTime=1732098417675; duration=0sec 2024-11-20T10:26:58,191 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:26:58,191 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b746a4c7c9036a37fd4050571abf4f3f:C 2024-11-20T10:26:58,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/a8f31681d1eb48768f3ed785315e5cf3 is 50, key is test_row_0/B:col10/1732098416890/Put/seqid=0 2024-11-20T10:26:58,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742402_1578 (size=12151) 2024-11-20T10:26:58,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098478377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:58,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098478377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:58,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098478377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T10:26:58,599 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/a8f31681d1eb48768f3ed785315e5cf3 2024-11-20T10:26:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/efe78a035ddc4574974ad04304a4082e is 50, key is test_row_0/C:col10/1732098416890/Put/seqid=0 2024-11-20T10:26:58,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742403_1579 (size=12151) 2024-11-20T10:26:58,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098478686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:58,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:58,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098478687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098478687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:59,021 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/efe78a035ddc4574974ad04304a4082e 2024-11-20T10:26:59,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/e798654fd20b47ae98617a27245e7f01 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/e798654fd20b47ae98617a27245e7f01 2024-11-20T10:26:59,027 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/e798654fd20b47ae98617a27245e7f01, entries=150, sequenceid=152, filesize=30.4 K 2024-11-20T10:26:59,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/a8f31681d1eb48768f3ed785315e5cf3 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/a8f31681d1eb48768f3ed785315e5cf3 2024-11-20T10:26:59,030 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/a8f31681d1eb48768f3ed785315e5cf3, entries=150, sequenceid=152, filesize=11.9 K 2024-11-20T10:26:59,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/efe78a035ddc4574974ad04304a4082e as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/efe78a035ddc4574974ad04304a4082e 2024-11-20T10:26:59,034 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/efe78a035ddc4574974ad04304a4082e, entries=150, sequenceid=152, filesize=11.9 K 2024-11-20T10:26:59,035 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for b746a4c7c9036a37fd4050571abf4f3f in 1317ms, sequenceid=152, compaction requested=false 2024-11-20T10:26:59,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:26:59,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:59,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-20T10:26:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-20T10:26:59,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-20T10:26:59,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6230 sec 2024-11-20T10:26:59,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.6320 sec 2024-11-20T10:26:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:59,200 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:26:59,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T10:26:59,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120214f1369698b48b2802623c2ee5cb034_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098418054/Put/seqid=0 2024-11-20T10:26:59,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098479230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:59,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:59,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098479231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:59,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098479232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:59,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742404_1580 (size=14794) 2024-11-20T10:26:59,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098479336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:59,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:59,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:59,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098479347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:59,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:59,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098479347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:59,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:59,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098479546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T10:26:59,557 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:26:59,557 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:26:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-20T10:26:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T10:26:59,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098479557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098479557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:59,560 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:26:59,561 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:26:59,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:26:59,626 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:26:59,629 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120214f1369698b48b2802623c2ee5cb034_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120214f1369698b48b2802623c2ee5cb034_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:26:59,630 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/567e88e68f9745dba75c0ce1c0b9296c, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:26:59,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/567e88e68f9745dba75c0ce1c0b9296c is 175, key is test_row_0/A:col10/1732098418054/Put/seqid=0 2024-11-20T10:26:59,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742405_1581 (size=39749) 2024-11-20T10:26:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T10:26:59,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T10:26:59,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:59,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:59,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:59,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:59,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:59,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098479856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:59,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:59,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T10:26:59,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:59,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:26:59,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:26:59,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:59,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:26:59,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098479867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:26:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098479867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:26:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:26:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:26:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T10:27:00,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T10:27:00,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:00,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,038 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/567e88e68f9745dba75c0ce1c0b9296c 2024-11-20T10:27:00,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/ec5705c1ff4b4c1781278304cab3ea2b is 50, key is test_row_0/B:col10/1732098418054/Put/seqid=0 2024-11-20T10:27:00,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742406_1582 (size=12151) 2024-11-20T10:27:00,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T10:27:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T10:27:00,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T10:27:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098480366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:00,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098480377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:00,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098480377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:00,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/ec5705c1ff4b4c1781278304cab3ea2b 2024-11-20T10:27:00,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/fb5359f5e7db4b0094029ea807d12fbb is 50, key is test_row_0/C:col10/1732098418054/Put/seqid=0 2024-11-20T10:27:00,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742407_1583 (size=12151) 2024-11-20T10:27:00,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T10:27:00,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:00,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T10:27:00,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:00,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T10:27:00,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T10:27:00,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:00,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:00,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/fb5359f5e7db4b0094029ea807d12fbb 2024-11-20T10:27:00,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/567e88e68f9745dba75c0ce1c0b9296c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/567e88e68f9745dba75c0ce1c0b9296c 2024-11-20T10:27:00,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/567e88e68f9745dba75c0ce1c0b9296c, entries=200, sequenceid=172, filesize=38.8 K 2024-11-20T10:27:00,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/ec5705c1ff4b4c1781278304cab3ea2b as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/ec5705c1ff4b4c1781278304cab3ea2b 2024-11-20T10:27:00,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/ec5705c1ff4b4c1781278304cab3ea2b, entries=150, sequenceid=172, filesize=11.9 K 2024-11-20T10:27:00,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/fb5359f5e7db4b0094029ea807d12fbb as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/fb5359f5e7db4b0094029ea807d12fbb 2024-11-20T10:27:00,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/fb5359f5e7db4b0094029ea807d12fbb, entries=150, sequenceid=172, filesize=11.9 K 2024-11-20T10:27:00,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for b746a4c7c9036a37fd4050571abf4f3f in 1669ms, sequenceid=172, compaction requested=true 2024-11-20T10:27:00,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:00,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b746a4c7c9036a37fd4050571abf4f3f:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:27:00,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:27:00,869 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:27:00,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b746a4c7c9036a37fd4050571abf4f3f:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:27:00,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:27:00,870 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:27:00,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b746a4c7c9036a37fd4050571abf4f3f:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:27:00,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:27:00,870 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102133 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:27:00,870 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:27:00,870 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): b746a4c7c9036a37fd4050571abf4f3f/A is initiating minor compaction (all files) 2024-11-20T10:27:00,870 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): b746a4c7c9036a37fd4050571abf4f3f/B is initiating minor compaction (all files) 2024-11-20T10:27:00,870 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b746a4c7c9036a37fd4050571abf4f3f/A in TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,870 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b746a4c7c9036a37fd4050571abf4f3f/B in TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,870 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/6887046602d04cc29dc231a934f487f4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/e798654fd20b47ae98617a27245e7f01, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/567e88e68f9745dba75c0ce1c0b9296c] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp, totalSize=99.7 K 2024-11-20T10:27:00,870 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/c1ce3da6a17441949dc8ecc971621a43, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/a8f31681d1eb48768f3ed785315e5cf3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/ec5705c1ff4b4c1781278304cab3ea2b] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp, totalSize=35.8 K 2024-11-20T10:27:00,870 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,870 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/6887046602d04cc29dc231a934f487f4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/e798654fd20b47ae98617a27245e7f01, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/567e88e68f9745dba75c0ce1c0b9296c] 2024-11-20T10:27:00,871 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c1ce3da6a17441949dc8ecc971621a43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732098415664 2024-11-20T10:27:00,871 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6887046602d04cc29dc231a934f487f4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732098415664 2024-11-20T10:27:00,871 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a8f31681d1eb48768f3ed785315e5cf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732098416884 2024-11-20T10:27:00,871 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e798654fd20b47ae98617a27245e7f01, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732098416884 2024-11-20T10:27:00,871 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ec5705c1ff4b4c1781278304cab3ea2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732098418048 2024-11-20T10:27:00,871 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 567e88e68f9745dba75c0ce1c0b9296c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732098418048 2024-11-20T10:27:00,876 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:00,877 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b746a4c7c9036a37fd4050571abf4f3f#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:27:00,877 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/5c1d1a65c89b461fa8535d78633ca53a is 50, key is test_row_0/B:col10/1732098418054/Put/seqid=0 2024-11-20T10:27:00,879 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120e4be2739dd654382bd1031c15bf5b41d_b746a4c7c9036a37fd4050571abf4f3f store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:00,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742408_1584 (size=12527) 2024-11-20T10:27:00,881 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120e4be2739dd654382bd1031c15bf5b41d_b746a4c7c9036a37fd4050571abf4f3f, store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:00,882 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e4be2739dd654382bd1031c15bf5b41d_b746a4c7c9036a37fd4050571abf4f3f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:00,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742409_1585 (size=4469) 2024-11-20T10:27:00,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T10:27:00,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:00,934 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T10:27:00,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203f18894113d544628a6de49dd4fa5bcc_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098419230/Put/seqid=0 2024-11-20T10:27:00,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742410_1586 (size=12304) 2024-11-20T10:27:01,286 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b746a4c7c9036a37fd4050571abf4f3f#A#compaction#493 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:27:01,287 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/51be02167cdf4410b83f16067b6e67c1 is 175, key is test_row_0/A:col10/1732098418054/Put/seqid=0 2024-11-20T10:27:01,288 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/5c1d1a65c89b461fa8535d78633ca53a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/5c1d1a65c89b461fa8535d78633ca53a 2024-11-20T10:27:01,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742411_1587 (size=31481) 2024-11-20T10:27:01,293 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b746a4c7c9036a37fd4050571abf4f3f/B of b746a4c7c9036a37fd4050571abf4f3f into 5c1d1a65c89b461fa8535d78633ca53a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:27:01,293 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:01,293 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., storeName=b746a4c7c9036a37fd4050571abf4f3f/B, priority=13, startTime=1732098420869; duration=0sec 2024-11-20T10:27:01,293 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:27:01,293 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b746a4c7c9036a37fd4050571abf4f3f:B 2024-11-20T10:27:01,293 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:27:01,294 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:27:01,294 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): b746a4c7c9036a37fd4050571abf4f3f/C is initiating minor compaction (all files) 2024-11-20T10:27:01,294 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b746a4c7c9036a37fd4050571abf4f3f/C in TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:01,294 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/4b941a2fa7e14d5b8f982ca9176897f0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/efe78a035ddc4574974ad04304a4082e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/fb5359f5e7db4b0094029ea807d12fbb] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp, totalSize=35.8 K 2024-11-20T10:27:01,294 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b941a2fa7e14d5b8f982ca9176897f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732098415664 2024-11-20T10:27:01,294 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting efe78a035ddc4574974ad04304a4082e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732098416884 2024-11-20T10:27:01,294 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting fb5359f5e7db4b0094029ea807d12fbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732098418048 2024-11-20T10:27:01,298 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b746a4c7c9036a37fd4050571abf4f3f#C#compaction#495 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:27:01,299 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/3ecffd10972c445a88733dd48a174755 is 50, key is test_row_0/C:col10/1732098418054/Put/seqid=0 2024-11-20T10:27:01,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742412_1588 (size=12527) 2024-11-20T10:27:01,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:01,345 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203f18894113d544628a6de49dd4fa5bcc_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203f18894113d544628a6de49dd4fa5bcc_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:01,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/fd6d846d19544497aec289615214b1da, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:01,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/fd6d846d19544497aec289615214b1da is 175, key is test_row_0/A:col10/1732098419230/Put/seqid=0 2024-11-20T10:27:01,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742413_1589 (size=31105) 2024-11-20T10:27:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:01,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:01,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098481415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:01,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:01,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098481415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:01,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:01,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:01,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098481415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:01,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:01,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:01,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098481527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:01,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098481527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:01,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098481527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:01,695 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/51be02167cdf4410b83f16067b6e67c1 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/51be02167cdf4410b83f16067b6e67c1 2024-11-20T10:27:01,698 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b746a4c7c9036a37fd4050571abf4f3f/A of b746a4c7c9036a37fd4050571abf4f3f into 51be02167cdf4410b83f16067b6e67c1(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:27:01,698 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:01,698 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., storeName=b746a4c7c9036a37fd4050571abf4f3f/A, priority=13, startTime=1732098420869; duration=0sec 2024-11-20T10:27:01,698 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:27:01,698 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b746a4c7c9036a37fd4050571abf4f3f:A 2024-11-20T10:27:01,704 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/3ecffd10972c445a88733dd48a174755 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/3ecffd10972c445a88733dd48a174755 2024-11-20T10:27:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T10:27:01,708 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b746a4c7c9036a37fd4050571abf4f3f/C of b746a4c7c9036a37fd4050571abf4f3f into 3ecffd10972c445a88733dd48a174755(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:27:01,708 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:01,708 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., storeName=b746a4c7c9036a37fd4050571abf4f3f/C, priority=13, startTime=1732098420870; duration=0sec 2024-11-20T10:27:01,708 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:27:01,708 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b746a4c7c9036a37fd4050571abf4f3f:C 2024-11-20T10:27:01,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098481736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:01,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098481737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:01,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:01,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:01,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098481737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:01,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:01,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:01,749 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/fd6d846d19544497aec289615214b1da 2024-11-20T10:27:01,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/8f8367216a444589ac7e05459145091d is 50, key is test_row_0/B:col10/1732098419230/Put/seqid=0 2024-11-20T10:27:01,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742414_1590 (size=12151) 2024-11-20T10:27:01,756 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/8f8367216a444589ac7e05459145091d 2024-11-20T10:27:01,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/20e8528122744543a5628d6a896d4975 is 50, key is test_row_0/C:col10/1732098419230/Put/seqid=0 2024-11-20T10:27:01,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742415_1591 (size=12151) 2024-11-20T10:27:02,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098482047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:02,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098482047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098482047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:02,165 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/20e8528122744543a5628d6a896d4975 2024-11-20T10:27:02,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/fd6d846d19544497aec289615214b1da as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/fd6d846d19544497aec289615214b1da 2024-11-20T10:27:02,171 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/fd6d846d19544497aec289615214b1da, entries=150, sequenceid=191, filesize=30.4 K 2024-11-20T10:27:02,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/8f8367216a444589ac7e05459145091d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/8f8367216a444589ac7e05459145091d 2024-11-20T10:27:02,175 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/8f8367216a444589ac7e05459145091d, entries=150, sequenceid=191, filesize=11.9 K 2024-11-20T10:27:02,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/20e8528122744543a5628d6a896d4975 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/20e8528122744543a5628d6a896d4975 2024-11-20T10:27:02,178 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/20e8528122744543a5628d6a896d4975, entries=150, sequenceid=191, filesize=11.9 K 2024-11-20T10:27:02,179 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for b746a4c7c9036a37fd4050571abf4f3f in 1245ms, sequenceid=191, compaction requested=false 2024-11-20T10:27:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-20T10:27:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-20T10:27:02,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-20T10:27:02,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6190 sec 2024-11-20T10:27:02,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.6250 sec 2024-11-20T10:27:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:02,559 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:27:02,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T10:27:02,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120592ad8a2ca0b498d80b6496796cee53a_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098422557/Put/seqid=0 2024-11-20T10:27:02,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742416_1592 (size=12304) 2024-11-20T10:27:02,577 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:02,579 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120592ad8a2ca0b498d80b6496796cee53a_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120592ad8a2ca0b498d80b6496796cee53a_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:02,580 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/c5a8c554bf4b480ca433a997910c01fc, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:02,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/c5a8c554bf4b480ca433a997910c01fc is 175, key is test_row_0/A:col10/1732098422557/Put/seqid=0 2024-11-20T10:27:02,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742417_1593 (size=31105) 2024-11-20T10:27:02,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098482586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:02,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:02,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098482589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:02,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:02,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:02,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098482590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:02,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098482696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:02,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:02,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098482696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:02,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:02,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098482696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:02,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:02,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098482907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098482907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:02,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098482907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:02,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:02,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:02,984 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/c5a8c554bf4b480ca433a997910c01fc 2024-11-20T10:27:02,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/cf592e4784314b02a56ec1e294d08d06 is 50, key is test_row_0/B:col10/1732098422557/Put/seqid=0 2024-11-20T10:27:02,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742418_1594 (size=12151) 2024-11-20T10:27:03,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098483216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:03,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:03,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098483216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:03,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:03,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:03,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098483217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:03,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:03,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/cf592e4784314b02a56ec1e294d08d06 2024-11-20T10:27:03,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/53dbea44b9af4e308d1bd97df5191ecf is 50, key is test_row_0/C:col10/1732098422557/Put/seqid=0 2024-11-20T10:27:03,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742419_1595 (size=12151) 2024-11-20T10:27:03,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/53dbea44b9af4e308d1bd97df5191ecf 2024-11-20T10:27:03,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/c5a8c554bf4b480ca433a997910c01fc as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/c5a8c554bf4b480ca433a997910c01fc 2024-11-20T10:27:03,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/c5a8c554bf4b480ca433a997910c01fc, entries=150, sequenceid=212, filesize=30.4 K 2024-11-20T10:27:03,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/cf592e4784314b02a56ec1e294d08d06 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/cf592e4784314b02a56ec1e294d08d06 2024-11-20T10:27:03,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/cf592e4784314b02a56ec1e294d08d06, entries=150, sequenceid=212, filesize=11.9 K 2024-11-20T10:27:03,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/53dbea44b9af4e308d1bd97df5191ecf as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/53dbea44b9af4e308d1bd97df5191ecf 2024-11-20T10:27:03,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/53dbea44b9af4e308d1bd97df5191ecf, entries=150, sequenceid=212, filesize=11.9 K 2024-11-20T10:27:03,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b746a4c7c9036a37fd4050571abf4f3f in 869ms, sequenceid=212, compaction requested=true 2024-11-20T10:27:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b746a4c7c9036a37fd4050571abf4f3f:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:27:03,428 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:27:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:27:03,428 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:27:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b746a4c7c9036a37fd4050571abf4f3f:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:27:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:27:03,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b746a4c7c9036a37fd4050571abf4f3f:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:27:03,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:27:03,429 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93691 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:27:03,429 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): b746a4c7c9036a37fd4050571abf4f3f/A is initiating minor compaction (all files) 2024-11-20T10:27:03,429 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b746a4c7c9036a37fd4050571abf4f3f/A in TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:03,429 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/51be02167cdf4410b83f16067b6e67c1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/fd6d846d19544497aec289615214b1da, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/c5a8c554bf4b480ca433a997910c01fc] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp, totalSize=91.5 K 2024-11-20T10:27:03,430 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:03,430 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/51be02167cdf4410b83f16067b6e67c1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/fd6d846d19544497aec289615214b1da, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/c5a8c554bf4b480ca433a997910c01fc] 2024-11-20T10:27:03,430 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:27:03,430 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): b746a4c7c9036a37fd4050571abf4f3f/B is initiating minor compaction (all files) 2024-11-20T10:27:03,430 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b746a4c7c9036a37fd4050571abf4f3f/B in TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:03,430 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/5c1d1a65c89b461fa8535d78633ca53a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/8f8367216a444589ac7e05459145091d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/cf592e4784314b02a56ec1e294d08d06] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp, totalSize=36.0 K 2024-11-20T10:27:03,430 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51be02167cdf4410b83f16067b6e67c1, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732098418048 2024-11-20T10:27:03,430 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c1d1a65c89b461fa8535d78633ca53a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732098418048 2024-11-20T10:27:03,431 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd6d846d19544497aec289615214b1da, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732098419229 2024-11-20T10:27:03,431 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f8367216a444589ac7e05459145091d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732098419229 2024-11-20T10:27:03,431 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5a8c554bf4b480ca433a997910c01fc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732098421403 2024-11-20T10:27:03,431 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting cf592e4784314b02a56ec1e294d08d06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732098421403 2024-11-20T10:27:03,436 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:03,437 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b746a4c7c9036a37fd4050571abf4f3f#B#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:27:03,437 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/0f7eea1c9b8a4eaf9a6d17b6ba2fe431 is 50, key is test_row_0/B:col10/1732098422557/Put/seqid=0 2024-11-20T10:27:03,448 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120eda52aeb50c444abb907f82ebb003594_b746a4c7c9036a37fd4050571abf4f3f store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:03,450 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120eda52aeb50c444abb907f82ebb003594_b746a4c7c9036a37fd4050571abf4f3f, store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:03,450 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120eda52aeb50c444abb907f82ebb003594_b746a4c7c9036a37fd4050571abf4f3f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:03,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742420_1596 (size=12629) 2024-11-20T10:27:03,471 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/0f7eea1c9b8a4eaf9a6d17b6ba2fe431 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/0f7eea1c9b8a4eaf9a6d17b6ba2fe431 2024-11-20T10:27:03,475 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b746a4c7c9036a37fd4050571abf4f3f/B of b746a4c7c9036a37fd4050571abf4f3f into 0f7eea1c9b8a4eaf9a6d17b6ba2fe431(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:27:03,475 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:03,475 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., storeName=b746a4c7c9036a37fd4050571abf4f3f/B, priority=13, startTime=1732098423428; duration=0sec 2024-11-20T10:27:03,475 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:27:03,475 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b746a4c7c9036a37fd4050571abf4f3f:B 2024-11-20T10:27:03,476 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T10:27:03,476 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T10:27:03,476 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): b746a4c7c9036a37fd4050571abf4f3f/C is initiating minor compaction (all files) 2024-11-20T10:27:03,476 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b746a4c7c9036a37fd4050571abf4f3f/C in TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:03,476 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/3ecffd10972c445a88733dd48a174755, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/20e8528122744543a5628d6a896d4975, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/53dbea44b9af4e308d1bd97df5191ecf] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp, totalSize=36.0 K 2024-11-20T10:27:03,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742421_1597 (size=4469) 2024-11-20T10:27:03,478 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ecffd10972c445a88733dd48a174755, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732098418048 2024-11-20T10:27:03,480 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 20e8528122744543a5628d6a896d4975, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732098419229 2024-11-20T10:27:03,480 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b746a4c7c9036a37fd4050571abf4f3f#A#compaction#502 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:27:03,481 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 53dbea44b9af4e308d1bd97df5191ecf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732098421403 2024-11-20T10:27:03,481 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/73c35edc1e8c45d29fd6784cb93d3f8c is 175, key is test_row_0/A:col10/1732098422557/Put/seqid=0 2024-11-20T10:27:03,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742422_1598 (size=31583) 2024-11-20T10:27:03,510 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b746a4c7c9036a37fd4050571abf4f3f#C#compaction#503 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:27:03,510 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/d0f85253627d452abd0a96fc3d00b9a7 is 50, key is test_row_0/C:col10/1732098422557/Put/seqid=0 2024-11-20T10:27:03,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742423_1599 (size=12629) 2024-11-20T10:27:03,534 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/d0f85253627d452abd0a96fc3d00b9a7 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/d0f85253627d452abd0a96fc3d00b9a7 2024-11-20T10:27:03,539 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b746a4c7c9036a37fd4050571abf4f3f/C of b746a4c7c9036a37fd4050571abf4f3f into d0f85253627d452abd0a96fc3d00b9a7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:27:03,539 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:03,539 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., storeName=b746a4c7c9036a37fd4050571abf4f3f/C, priority=13, startTime=1732098423428; duration=0sec 2024-11-20T10:27:03,539 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:27:03,539 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b746a4c7c9036a37fd4050571abf4f3f:C 2024-11-20T10:27:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T10:27:03,727 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:27:03,727 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:27:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-20T10:27:03,728 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:27:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T10:27:03,729 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:27:03,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:27:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:03,731 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:27:03,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T10:27:03,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202d314e4d057247399b945100e5167a53_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098422583/Put/seqid=0 2024-11-20T10:27:03,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742424_1600 (size=12304) 2024-11-20T10:27:03,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098483756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:03,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098483757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098483757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T10:27:03,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098483867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:03,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098483867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:03,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098483867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:03,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T10:27:03,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:03,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:03,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:03,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:03,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:03,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:03,900 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/73c35edc1e8c45d29fd6784cb93d3f8c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/73c35edc1e8c45d29fd6784cb93d3f8c 2024-11-20T10:27:03,904 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b746a4c7c9036a37fd4050571abf4f3f/A of b746a4c7c9036a37fd4050571abf4f3f into 73c35edc1e8c45d29fd6784cb93d3f8c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:27:03,904 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:03,904 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., storeName=b746a4c7c9036a37fd4050571abf4f3f/A, priority=13, startTime=1732098423428; duration=0sec 2024-11-20T10:27:03,904 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:27:03,904 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b746a4c7c9036a37fd4050571abf4f3f:A 2024-11-20T10:27:04,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T10:27:04,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:04,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:04,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:04,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T10:27:04,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098484077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:04,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098484077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:04,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098484084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:04,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:04,154 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:04,157 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202d314e4d057247399b945100e5167a53_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202d314e4d057247399b945100e5167a53_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:04,157 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/75119dd473394ba98ec5f5581f5a4b90, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:04,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/75119dd473394ba98ec5f5581f5a4b90 is 175, key is test_row_0/A:col10/1732098422583/Put/seqid=0 2024-11-20T10:27:04,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742425_1601 (size=31105) 2024-11-20T10:27:04,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T10:27:04,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:04,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:04,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:04,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T10:27:04,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:04,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:04,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:04,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T10:27:04,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098484387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:04,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098484387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:04,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098484396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:04,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T10:27:04,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:04,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:04,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:04,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,568 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/75119dd473394ba98ec5f5581f5a4b90 2024-11-20T10:27:04,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/d5eaeae1c1684968b914d41d3e750483 is 50, key is test_row_0/B:col10/1732098422583/Put/seqid=0 2024-11-20T10:27:04,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742426_1602 (size=12151) 2024-11-20T10:27:04,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T10:27:04,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:04,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:04,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:04,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T10:27:04,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:04,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:04,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:04,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T10:27:04,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:04,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098484896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:04,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:04,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098484896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:04,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:04,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:04,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098484907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:04,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T10:27:04,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:04,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:04,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:04,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:04,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/d5eaeae1c1684968b914d41d3e750483 2024-11-20T10:27:04,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/7f35a57cd42a4d6faf21e729aa397765 is 50, key is test_row_0/C:col10/1732098422583/Put/seqid=0 2024-11-20T10:27:04,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742427_1603 (size=12151) 2024-11-20T10:27:04,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/7f35a57cd42a4d6faf21e729aa397765 2024-11-20T10:27:05,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/75119dd473394ba98ec5f5581f5a4b90 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/75119dd473394ba98ec5f5581f5a4b90 2024-11-20T10:27:05,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/75119dd473394ba98ec5f5581f5a4b90, entries=150, sequenceid=235, filesize=30.4 K 2024-11-20T10:27:05,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/d5eaeae1c1684968b914d41d3e750483 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/d5eaeae1c1684968b914d41d3e750483 2024-11-20T10:27:05,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/d5eaeae1c1684968b914d41d3e750483, entries=150, sequenceid=235, filesize=11.9 K 2024-11-20T10:27:05,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/7f35a57cd42a4d6faf21e729aa397765 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/7f35a57cd42a4d6faf21e729aa397765 2024-11-20T10:27:05,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/7f35a57cd42a4d6faf21e729aa397765, entries=150, sequenceid=235, filesize=11.9 K 2024-11-20T10:27:05,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for b746a4c7c9036a37fd4050571abf4f3f in 1281ms, sequenceid=235, compaction requested=false 2024-11-20T10:27:05,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:05,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T10:27:05,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:05,116 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-20T10:27:05,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112051d0179bc2984e1e91eaae16aa3c6e73_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098423755/Put/seqid=0 2024-11-20T10:27:05,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742428_1604 (size=12304) 2024-11-20T10:27:05,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,130 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112051d0179bc2984e1e91eaae16aa3c6e73_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112051d0179bc2984e1e91eaae16aa3c6e73_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:05,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/1c43be542ee44e1384c353fe2d4fb633, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:05,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/1c43be542ee44e1384c353fe2d4fb633 is 175, key is test_row_0/A:col10/1732098423755/Put/seqid=0 2024-11-20T10:27:05,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742429_1605 (size=31105) 2024-11-20T10:27:05,135 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/1c43be542ee44e1384c353fe2d4fb633 2024-11-20T10:27:05,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/0ff0fbd6d5fa413ebe8cce070b1864c0 is 50, key is test_row_0/B:col10/1732098423755/Put/seqid=0 2024-11-20T10:27:05,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742430_1606 (size=12151) 2024-11-20T10:27:05,545 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/0ff0fbd6d5fa413ebe8cce070b1864c0 2024-11-20T10:27:05,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/542e8f4714b146898d399a874da7be0c is 50, key is test_row_0/C:col10/1732098423755/Put/seqid=0 2024-11-20T10:27:05,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742431_1607 (size=12151) 2024-11-20T10:27:05,558 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/542e8f4714b146898d399a874da7be0c 2024-11-20T10:27:05,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/1c43be542ee44e1384c353fe2d4fb633 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/1c43be542ee44e1384c353fe2d4fb633 2024-11-20T10:27:05,579 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/1c43be542ee44e1384c353fe2d4fb633, entries=150, sequenceid=251, filesize=30.4 K 2024-11-20T10:27:05,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/0ff0fbd6d5fa413ebe8cce070b1864c0 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/0ff0fbd6d5fa413ebe8cce070b1864c0 2024-11-20T10:27:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,583 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/0ff0fbd6d5fa413ebe8cce070b1864c0, entries=150, sequenceid=251, filesize=11.9 K 2024-11-20T10:27:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/542e8f4714b146898d399a874da7be0c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/542e8f4714b146898d399a874da7be0c 2024-11-20T10:27:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,587 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/542e8f4714b146898d399a874da7be0c, entries=150, sequenceid=251, filesize=11.9 K 2024-11-20T10:27:05,588 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for b746a4c7c9036a37fd4050571abf4f3f in 472ms, sequenceid=251, compaction requested=true 2024-11-20T10:27:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-20T10:27:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-20T10:27:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-20T10:27:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8600 sec 2024-11-20T10:27:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.8640 sec 2024-11-20T10:27:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:05,811 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T10:27:05,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T10:27:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205e90ec590c984f90812f5fe38b4aec0a_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098425811/Put/seqid=0 2024-11-20T10:27:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:05,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742432_1608 (size=20074) 2024-11-20T10:27:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T10:27:05,877 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:27:05,878 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T10:27:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-20T10:27:05,880 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T10:27:05,881 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T10:27:05,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T10:27:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T10:27:05,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098485893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:05,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098485896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:05,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42422 deadline: 1732098485906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:05,909 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_0' in b746a4c7c9036a37fd4050571abf4f3f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:27:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:05,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42428 deadline: 1732098485917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:05,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42444 deadline: 1732098485917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:05,919 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_0' in b746a4c7c9036a37fd4050571abf4f3f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:27:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:05,919 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 8c533afd331c,35833,1732098238678 for 'test_row_2' in b746a4c7c9036a37fd4050571abf4f3f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T10:27:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T10:27:06,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098486006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:06,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:06,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:06,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:06,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098486006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:06,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:06,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:06,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:06,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T10:27:06,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:06,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:06,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:06,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T10:27:06,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:06,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:06,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:06,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T10:27:06,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098486216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:06,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098486216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:06,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:06,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:06,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:06,254 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:06,256 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205e90ec590c984f90812f5fe38b4aec0a_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205e90ec590c984f90812f5fe38b4aec0a_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:06,257 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/0855495d53c742ef9d0092bc24eb2588, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:06,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/0855495d53c742ef9d0092bc24eb2588 is 175, key is test_row_0/A:col10/1732098425811/Put/seqid=0 2024-11-20T10:27:06,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742433_1609 (size=57333) 2024-11-20T10:27:06,260 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/0855495d53c742ef9d0092bc24eb2588 2024-11-20T10:27:06,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/fb82b7de834d49c78379eb08f58bcd24 is 50, key is test_row_0/B:col10/1732098425811/Put/seqid=0 2024-11-20T10:27:06,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742434_1610 (size=12301) 2024-11-20T10:27:06,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T10:27:06,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:06,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:06,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:06,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T10:27:06,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:06,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:06,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:06,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T10:27:06,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098486526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:06,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098486527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:06,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T10:27:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:06,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:06,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:06,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/fb82b7de834d49c78379eb08f58bcd24 2024-11-20T10:27:06,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/c69544cae6684116a21371bf54709501 is 50, key is test_row_0/C:col10/1732098425811/Put/seqid=0 2024-11-20T10:27:06,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742435_1611 (size=12301) 2024-11-20T10:27:06,779 INFO [Thread-2422 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2422. 2024-11-20T10:27:06,779 DEBUG [Thread-2422 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:27:06,779 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:06,779 INFO [Thread-2424 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2424. 2024-11-20T10:27:06,779 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:06,779 DEBUG [Thread-2424 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:27:06,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:27:06,779 DEBUG [Thread-2424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:06,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:27:06,780 DEBUG [Thread-2424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:06,780 INFO [Thread-2428 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2428. 2024-11-20T10:27:06,780 DEBUG [Thread-2428 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:27:06,780 DEBUG [Thread-2428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:06,780 DEBUG [Thread-2428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:06,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:27:06,783 INFO [Thread-2426 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2426. 2024-11-20T10:27:06,783 DEBUG [Thread-2426 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:27:06,783 DEBUG [Thread-2426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:06,783 DEBUG [Thread-2426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:06,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:27:06,783 INFO [Thread-2430 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2430. 2024-11-20T10:27:06,783 DEBUG [Thread-2430 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:27:06,783 DEBUG [Thread-2430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:06,783 DEBUG [Thread-2430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:06,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:27:06,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T10:27:06,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:06,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:06,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:06,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T10:27:06,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:06,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. as already flushing 2024-11-20T10:27:06,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:06,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:27:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T10:27:07,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:07,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T10:27:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1732098487037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42408 deadline: 1732098487037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 2024-11-20T10:27:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 , the old value is region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:27:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., hostname=8c533afd331c,35833,1732098238678, seqNum=5 because the exception is null or not the one we care about 2024-11-20T10:27:07,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/c69544cae6684116a21371bf54709501 2024-11-20T10:27:07,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/0855495d53c742ef9d0092bc24eb2588 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/0855495d53c742ef9d0092bc24eb2588 2024-11-20T10:27:07,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/0855495d53c742ef9d0092bc24eb2588, entries=300, sequenceid=263, filesize=56.0 K 2024-11-20T10:27:07,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/fb82b7de834d49c78379eb08f58bcd24 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/fb82b7de834d49c78379eb08f58bcd24 2024-11-20T10:27:07,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/fb82b7de834d49c78379eb08f58bcd24, entries=150, sequenceid=263, filesize=12.0 K 2024-11-20T10:27:07,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/c69544cae6684116a21371bf54709501 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/c69544cae6684116a21371bf54709501 2024-11-20T10:27:07,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/c69544cae6684116a21371bf54709501, entries=150, sequenceid=263, filesize=12.0 K 2024-11-20T10:27:07,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b746a4c7c9036a37fd4050571abf4f3f in 1292ms, sequenceid=263, compaction requested=true 2024-11-20T10:27:07,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:07,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b746a4c7c9036a37fd4050571abf4f3f:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T10:27:07,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:27:07,103 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:27:07,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b746a4c7c9036a37fd4050571abf4f3f:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T10:27:07,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:27:07,103 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:27:07,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b746a4c7c9036a37fd4050571abf4f3f:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T10:27:07,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:27:07,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T10:27:07,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:07,104 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T10:27:07,104 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151126 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:27:07,104 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1541): b746a4c7c9036a37fd4050571abf4f3f/A is initiating minor compaction (all files) 2024-11-20T10:27:07,104 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:27:07,104 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b746a4c7c9036a37fd4050571abf4f3f/A in TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:07,104 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): b746a4c7c9036a37fd4050571abf4f3f/B is initiating minor compaction (all files) 2024-11-20T10:27:07,104 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/73c35edc1e8c45d29fd6784cb93d3f8c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/75119dd473394ba98ec5f5581f5a4b90, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/1c43be542ee44e1384c353fe2d4fb633, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/0855495d53c742ef9d0092bc24eb2588] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp, totalSize=147.6 K 2024-11-20T10:27:07,104 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b746a4c7c9036a37fd4050571abf4f3f/B in TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:07,104 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:07,105 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/0f7eea1c9b8a4eaf9a6d17b6ba2fe431, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/d5eaeae1c1684968b914d41d3e750483, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/0ff0fbd6d5fa413ebe8cce070b1864c0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/fb82b7de834d49c78379eb08f58bcd24] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp, totalSize=48.1 K 2024-11-20T10:27:07,105 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. files: [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/73c35edc1e8c45d29fd6784cb93d3f8c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/75119dd473394ba98ec5f5581f5a4b90, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/1c43be542ee44e1384c353fe2d4fb633, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/0855495d53c742ef9d0092bc24eb2588] 2024-11-20T10:27:07,105 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f7eea1c9b8a4eaf9a6d17b6ba2fe431, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732098421403 2024-11-20T10:27:07,105 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73c35edc1e8c45d29fd6784cb93d3f8c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732098421403 2024-11-20T10:27:07,105 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d5eaeae1c1684968b914d41d3e750483, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732098422583 2024-11-20T10:27:07,105 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75119dd473394ba98ec5f5581f5a4b90, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732098422583 2024-11-20T10:27:07,105 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ff0fbd6d5fa413ebe8cce070b1864c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732098423754 2024-11-20T10:27:07,105 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c43be542ee44e1384c353fe2d4fb633, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732098423754 2024-11-20T10:27:07,106 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting fb82b7de834d49c78379eb08f58bcd24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732098425797 2024-11-20T10:27:07,106 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0855495d53c742ef9d0092bc24eb2588, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732098425796 2024-11-20T10:27:07,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207f0a2da89c6844a9a9b19eefb5f38538_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098425895/Put/seqid=0 2024-11-20T10:27:07,114 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:07,114 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b746a4c7c9036a37fd4050571abf4f3f#B#compaction#514 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:27:07,115 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/dea53077c2db4bbbac4c74f2f973ea65 is 50, key is test_row_0/B:col10/1732098425811/Put/seqid=0 2024-11-20T10:27:07,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742436_1612 (size=12454) 2024-11-20T10:27:07,116 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120fc69cf390a6d444d8e9016c9da9c39b0_b746a4c7c9036a37fd4050571abf4f3f store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:07,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742437_1613 (size=12915) 2024-11-20T10:27:07,125 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120fc69cf390a6d444d8e9016c9da9c39b0_b746a4c7c9036a37fd4050571abf4f3f, store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:07,125 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120fc69cf390a6d444d8e9016c9da9c39b0_b746a4c7c9036a37fd4050571abf4f3f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:07,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742438_1614 (size=4469) 2024-11-20T10:27:07,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:07,523 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207f0a2da89c6844a9a9b19eefb5f38538_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207f0a2da89c6844a9a9b19eefb5f38538_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:07,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/10a4a9545d90423da8da4a1126e18886, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:07,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/10a4a9545d90423da8da4a1126e18886 is 175, key is test_row_0/A:col10/1732098425895/Put/seqid=0 2024-11-20T10:27:07,528 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/dea53077c2db4bbbac4c74f2f973ea65 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/dea53077c2db4bbbac4c74f2f973ea65 2024-11-20T10:27:07,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742439_1615 (size=31255) 2024-11-20T10:27:07,529 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/10a4a9545d90423da8da4a1126e18886 2024-11-20T10:27:07,532 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b746a4c7c9036a37fd4050571abf4f3f/B of b746a4c7c9036a37fd4050571abf4f3f into dea53077c2db4bbbac4c74f2f973ea65(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:27:07,532 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:07,532 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., storeName=b746a4c7c9036a37fd4050571abf4f3f/B, priority=12, startTime=1732098427103; duration=0sec 2024-11-20T10:27:07,533 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T10:27:07,533 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b746a4c7c9036a37fd4050571abf4f3f:B 2024-11-20T10:27:07,533 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:27:07,534 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T10:27:07,534 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1541): b746a4c7c9036a37fd4050571abf4f3f/C is initiating minor compaction (all files) 2024-11-20T10:27:07,534 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b746a4c7c9036a37fd4050571abf4f3f/C in TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:07,534 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/d0f85253627d452abd0a96fc3d00b9a7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/7f35a57cd42a4d6faf21e729aa397765, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/542e8f4714b146898d399a874da7be0c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/c69544cae6684116a21371bf54709501] into tmpdir=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp, totalSize=48.1 K 2024-11-20T10:27:07,534 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d0f85253627d452abd0a96fc3d00b9a7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732098421403 2024-11-20T10:27:07,534 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f35a57cd42a4d6faf21e729aa397765, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732098422583 2024-11-20T10:27:07,535 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 542e8f4714b146898d399a874da7be0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732098423754 2024-11-20T10:27:07,535 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c69544cae6684116a21371bf54709501, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732098425797 2024-11-20T10:27:07,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/aba5117eb05f47c097d7629aa215049c is 50, key is test_row_0/B:col10/1732098425895/Put/seqid=0 2024-11-20T10:27:07,537 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b746a4c7c9036a37fd4050571abf4f3f#A#compaction#515 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:27:07,538 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/a117167e4ba3419ca0d7c14b8678ee13 is 175, key is test_row_0/A:col10/1732098425811/Put/seqid=0 2024-11-20T10:27:07,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742440_1616 (size=12301) 2024-11-20T10:27:07,539 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/aba5117eb05f47c097d7629aa215049c 2024-11-20T10:27:07,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742441_1617 (size=31869) 2024-11-20T10:27:07,544 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b746a4c7c9036a37fd4050571abf4f3f#C#compaction#517 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:27:07,544 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/1b649bdfa1e6450d9751fbe63dd8815d is 50, key is test_row_0/C:col10/1732098425811/Put/seqid=0 2024-11-20T10:27:07,545 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/a117167e4ba3419ca0d7c14b8678ee13 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/a117167e4ba3419ca0d7c14b8678ee13 2024-11-20T10:27:07,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/bfa084981f2240379a8e612554ec403c is 50, key is test_row_0/C:col10/1732098425895/Put/seqid=0 2024-11-20T10:27:07,549 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b746a4c7c9036a37fd4050571abf4f3f/A of b746a4c7c9036a37fd4050571abf4f3f into a117167e4ba3419ca0d7c14b8678ee13(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:27:07,550 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:07,550 INFO [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., storeName=b746a4c7c9036a37fd4050571abf4f3f/A, priority=12, startTime=1732098427103; duration=0sec 2024-11-20T10:27:07,550 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:27:07,550 DEBUG [RS:0;8c533afd331c:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b746a4c7c9036a37fd4050571abf4f3f:A 2024-11-20T10:27:07,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742442_1618 (size=12915) 2024-11-20T10:27:07,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742443_1619 (size=12301) 2024-11-20T10:27:07,552 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/bfa084981f2240379a8e612554ec403c 2024-11-20T10:27:07,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/10a4a9545d90423da8da4a1126e18886 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/10a4a9545d90423da8da4a1126e18886 2024-11-20T10:27:07,558 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/10a4a9545d90423da8da4a1126e18886, entries=150, sequenceid=287, filesize=30.5 K 2024-11-20T10:27:07,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/aba5117eb05f47c097d7629aa215049c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/aba5117eb05f47c097d7629aa215049c 2024-11-20T10:27:07,561 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/aba5117eb05f47c097d7629aa215049c, entries=150, sequenceid=287, filesize=12.0 K 2024-11-20T10:27:07,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/bfa084981f2240379a8e612554ec403c as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/bfa084981f2240379a8e612554ec403c 2024-11-20T10:27:07,564 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/bfa084981f2240379a8e612554ec403c, entries=150, sequenceid=287, filesize=12.0 K 2024-11-20T10:27:07,565 INFO [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for b746a4c7c9036a37fd4050571abf4f3f in 461ms, sequenceid=287, compaction requested=false 2024-11-20T10:27:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-20T10:27:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-20T10:27:07,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-20T10:27:07,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6850 sec 2024-11-20T10:27:07,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.6900 sec 2024-11-20T10:27:07,932 INFO [Thread-2415 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2415. 2024-11-20T10:27:07,932 INFO [Thread-2411 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2411. 2024-11-20T10:27:07,932 DEBUG [Thread-2415 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:27:07,932 DEBUG [Thread-2411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:27:07,932 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:07,932 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:07,933 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:07,933 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:07,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:27:07,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:27:07,938 INFO [Thread-2413 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2413. 2024-11-20T10:27:07,938 DEBUG [Thread-2413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:27:07,938 DEBUG [Thread-2413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:07,938 DEBUG [Thread-2413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:07,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:27:07,957 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/1b649bdfa1e6450d9751fbe63dd8815d as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/1b649bdfa1e6450d9751fbe63dd8815d 2024-11-20T10:27:07,963 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b746a4c7c9036a37fd4050571abf4f3f/C of b746a4c7c9036a37fd4050571abf4f3f into 1b649bdfa1e6450d9751fbe63dd8815d(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:27:07,963 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b746a4c7c9036a37fd4050571abf4f3f: 2024-11-20T10:27:07,963 INFO [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., storeName=b746a4c7c9036a37fd4050571abf4f3f/C, priority=12, startTime=1732098427103; duration=0sec 2024-11-20T10:27:07,963 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T10:27:07,963 DEBUG [RS:0;8c533afd331c:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b746a4c7c9036a37fd4050571abf4f3f:C 2024-11-20T10:27:08,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T10:27:08,027 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:27:08,050 INFO [Thread-2417 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2417. 2024-11-20T10:27:08,050 DEBUG [Thread-2417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:27:08,050 DEBUG [Thread-2417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:08,050 DEBUG [Thread-2417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:08,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:27:08,059 INFO [Thread-2419 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2419. 2024-11-20T10:27:08,059 DEBUG [Thread-2419 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T10:27:08,059 DEBUG [Thread-2419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:08,059 DEBUG [Thread-2419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:08,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:27:08,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T10:27:08,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-20T10:27:08,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-20T10:27:08,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-20T10:27:08,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-20T10:27:08,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-20T10:27:08,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T10:27:08,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3507 2024-11-20T10:27:08,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3497 2024-11-20T10:27:08,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3555 2024-11-20T10:27:08,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3508 2024-11-20T10:27:08,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3495 2024-11-20T10:27:08,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T10:27:08,061 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T10:27:08,061 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T10:27:08,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:08,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:08,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:27:08,062 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T10:27:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T10:27:08,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T10:27:08,068 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098428068"}]},"ts":"1732098428068"} 2024-11-20T10:27:08,070 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T10:27:08,071 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T10:27:08,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T10:27:08,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b746a4c7c9036a37fd4050571abf4f3f, UNASSIGN}] 2024-11-20T10:27:08,075 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b746a4c7c9036a37fd4050571abf4f3f, UNASSIGN 2024-11-20T10:27:08,076 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=b746a4c7c9036a37fd4050571abf4f3f, regionState=CLOSING, regionLocation=8c533afd331c,35833,1732098238678 2024-11-20T10:27:08,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b746a4c7c9036a37fd4050571abf4f3f, UNASSIGN because future has completed 2024-11-20T10:27:08,078 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T10:27:08,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678}] 2024-11-20T10:27:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T10:27:08,229 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:08,229 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T10:27:08,230 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing b746a4c7c9036a37fd4050571abf4f3f, disabling compactions & flushes 2024-11-20T10:27:08,230 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:08,230 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:08,230 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. after waiting 0 ms 2024-11-20T10:27:08,230 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:08,230 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing b746a4c7c9036a37fd4050571abf4f3f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T10:27:08,236 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120245c90beddbe42499fc1edba37852423_b746a4c7c9036a37fd4050571abf4f3f is 50, key is test_row_0/A:col10/1732098428048/Put/seqid=0 2024-11-20T10:27:08,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742444_1620 (size=9914) 2024-11-20T10:27:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T10:27:08,642 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:27:08,651 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120245c90beddbe42499fc1edba37852423_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120245c90beddbe42499fc1edba37852423_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:08,654 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/f392705e2c4b4482829939d69c402062, store: [table=TestAcidGuarantees family=A region=b746a4c7c9036a37fd4050571abf4f3f] 2024-11-20T10:27:08,655 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/f392705e2c4b4482829939d69c402062 is 175, key is test_row_0/A:col10/1732098428048/Put/seqid=0 2024-11-20T10:27:08,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742445_1621 (size=22561) 2024-11-20T10:27:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T10:27:09,062 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/f392705e2c4b4482829939d69c402062 2024-11-20T10:27:09,067 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/d0deadcdc546472b8a04ea03309cc6d1 is 50, key is test_row_0/B:col10/1732098428048/Put/seqid=0 2024-11-20T10:27:09,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742446_1622 (size=9857) 2024-11-20T10:27:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T10:27:09,471 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/d0deadcdc546472b8a04ea03309cc6d1 2024-11-20T10:27:09,485 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/402ccb65c1b1474bbc6557d586be9db9 is 50, key is test_row_0/C:col10/1732098428048/Put/seqid=0 2024-11-20T10:27:09,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742447_1623 (size=9857) 2024-11-20T10:27:09,890 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/402ccb65c1b1474bbc6557d586be9db9 2024-11-20T10:27:09,901 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/A/f392705e2c4b4482829939d69c402062 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/f392705e2c4b4482829939d69c402062 2024-11-20T10:27:09,904 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/f392705e2c4b4482829939d69c402062, entries=100, sequenceid=298, filesize=22.0 K 2024-11-20T10:27:09,905 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/B/d0deadcdc546472b8a04ea03309cc6d1 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/d0deadcdc546472b8a04ea03309cc6d1 2024-11-20T10:27:09,909 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/d0deadcdc546472b8a04ea03309cc6d1, entries=100, sequenceid=298, filesize=9.6 K 2024-11-20T10:27:09,909 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/.tmp/C/402ccb65c1b1474bbc6557d586be9db9 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/402ccb65c1b1474bbc6557d586be9db9 2024-11-20T10:27:09,913 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/402ccb65c1b1474bbc6557d586be9db9, entries=100, sequenceid=298, filesize=9.6 K 2024-11-20T10:27:09,914 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for b746a4c7c9036a37fd4050571abf4f3f in 1684ms, sequenceid=298, compaction requested=true 2024-11-20T10:27:09,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/bb029c70380a4f87aec9152dc8e7b215, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/c51eab12846748d79cbef1f8e7212c5f, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/263865b9920a4000b10c4a76ce8836aa, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/2503df1e57ba48939e2074f35b14eda7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/eee1474081ae45f192fe80239e0fdfd9, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/7add99ae38a0412b920964bc2f3c8249, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/9bddc6219ad2474db519b12f5bd4eda4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/b3cf24387bd8492cb670c0b841758f0e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/6887046602d04cc29dc231a934f487f4, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/e798654fd20b47ae98617a27245e7f01, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/567e88e68f9745dba75c0ce1c0b9296c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/51be02167cdf4410b83f16067b6e67c1, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/fd6d846d19544497aec289615214b1da, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/73c35edc1e8c45d29fd6784cb93d3f8c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/c5a8c554bf4b480ca433a997910c01fc, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/75119dd473394ba98ec5f5581f5a4b90, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/1c43be542ee44e1384c353fe2d4fb633, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/0855495d53c742ef9d0092bc24eb2588] to archive 2024-11-20T10:27:09,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:27:09,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/bb029c70380a4f87aec9152dc8e7b215 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/bb029c70380a4f87aec9152dc8e7b215 2024-11-20T10:27:09,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/c51eab12846748d79cbef1f8e7212c5f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/c51eab12846748d79cbef1f8e7212c5f 2024-11-20T10:27:09,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/263865b9920a4000b10c4a76ce8836aa to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/263865b9920a4000b10c4a76ce8836aa 2024-11-20T10:27:09,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/2503df1e57ba48939e2074f35b14eda7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/2503df1e57ba48939e2074f35b14eda7 2024-11-20T10:27:09,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/eee1474081ae45f192fe80239e0fdfd9 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/eee1474081ae45f192fe80239e0fdfd9 2024-11-20T10:27:09,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/7add99ae38a0412b920964bc2f3c8249 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/7add99ae38a0412b920964bc2f3c8249 2024-11-20T10:27:09,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/9bddc6219ad2474db519b12f5bd4eda4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/9bddc6219ad2474db519b12f5bd4eda4 2024-11-20T10:27:09,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/b3cf24387bd8492cb670c0b841758f0e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/b3cf24387bd8492cb670c0b841758f0e 2024-11-20T10:27:09,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/6887046602d04cc29dc231a934f487f4 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/6887046602d04cc29dc231a934f487f4 2024-11-20T10:27:09,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/e798654fd20b47ae98617a27245e7f01 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/e798654fd20b47ae98617a27245e7f01 2024-11-20T10:27:09,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/567e88e68f9745dba75c0ce1c0b9296c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/567e88e68f9745dba75c0ce1c0b9296c 2024-11-20T10:27:09,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/51be02167cdf4410b83f16067b6e67c1 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/51be02167cdf4410b83f16067b6e67c1 2024-11-20T10:27:09,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/fd6d846d19544497aec289615214b1da to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/fd6d846d19544497aec289615214b1da 2024-11-20T10:27:09,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/73c35edc1e8c45d29fd6784cb93d3f8c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/73c35edc1e8c45d29fd6784cb93d3f8c 2024-11-20T10:27:09,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/c5a8c554bf4b480ca433a997910c01fc to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/c5a8c554bf4b480ca433a997910c01fc 2024-11-20T10:27:09,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/75119dd473394ba98ec5f5581f5a4b90 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/75119dd473394ba98ec5f5581f5a4b90 2024-11-20T10:27:09,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/1c43be542ee44e1384c353fe2d4fb633 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/1c43be542ee44e1384c353fe2d4fb633 2024-11-20T10:27:09,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/0855495d53c742ef9d0092bc24eb2588 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/0855495d53c742ef9d0092bc24eb2588 2024-11-20T10:27:09,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/7de5dec6a84f46198c2f1f07c5899424, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/455488e4bf4e4bcfa906d1daadb2b89b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/39ece95df412406a81ad5ca23e0bcfc8, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/34568215e7f240458ba1f20eb35ca7cf, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/aea684b389014ce7940c983fc6187d46, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/7544e4aff07c496980d55d61e6912c61, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/486eb2c38eb24fdc9cb90262dc3efda6, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/c1ce3da6a17441949dc8ecc971621a43, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/268bed34008f42e6b87298e09bb6b04a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/a8f31681d1eb48768f3ed785315e5cf3, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/5c1d1a65c89b461fa8535d78633ca53a, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/ec5705c1ff4b4c1781278304cab3ea2b, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/8f8367216a444589ac7e05459145091d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/0f7eea1c9b8a4eaf9a6d17b6ba2fe431, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/cf592e4784314b02a56ec1e294d08d06, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/d5eaeae1c1684968b914d41d3e750483, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/0ff0fbd6d5fa413ebe8cce070b1864c0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/fb82b7de834d49c78379eb08f58bcd24] to archive 2024-11-20T10:27:09,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:27:09,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/7de5dec6a84f46198c2f1f07c5899424 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/7de5dec6a84f46198c2f1f07c5899424 2024-11-20T10:27:09,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/455488e4bf4e4bcfa906d1daadb2b89b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/455488e4bf4e4bcfa906d1daadb2b89b 2024-11-20T10:27:09,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/39ece95df412406a81ad5ca23e0bcfc8 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/39ece95df412406a81ad5ca23e0bcfc8 2024-11-20T10:27:09,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/34568215e7f240458ba1f20eb35ca7cf to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/34568215e7f240458ba1f20eb35ca7cf 2024-11-20T10:27:09,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/aea684b389014ce7940c983fc6187d46 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/aea684b389014ce7940c983fc6187d46 2024-11-20T10:27:09,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/7544e4aff07c496980d55d61e6912c61 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/7544e4aff07c496980d55d61e6912c61 2024-11-20T10:27:09,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/486eb2c38eb24fdc9cb90262dc3efda6 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/486eb2c38eb24fdc9cb90262dc3efda6 2024-11-20T10:27:09,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/c1ce3da6a17441949dc8ecc971621a43 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/c1ce3da6a17441949dc8ecc971621a43 2024-11-20T10:27:09,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/268bed34008f42e6b87298e09bb6b04a to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/268bed34008f42e6b87298e09bb6b04a 2024-11-20T10:27:09,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/a8f31681d1eb48768f3ed785315e5cf3 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/a8f31681d1eb48768f3ed785315e5cf3 2024-11-20T10:27:09,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/5c1d1a65c89b461fa8535d78633ca53a to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/5c1d1a65c89b461fa8535d78633ca53a 2024-11-20T10:27:09,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/ec5705c1ff4b4c1781278304cab3ea2b to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/ec5705c1ff4b4c1781278304cab3ea2b 2024-11-20T10:27:09,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/8f8367216a444589ac7e05459145091d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/8f8367216a444589ac7e05459145091d 2024-11-20T10:27:09,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/0f7eea1c9b8a4eaf9a6d17b6ba2fe431 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/0f7eea1c9b8a4eaf9a6d17b6ba2fe431 2024-11-20T10:27:09,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/cf592e4784314b02a56ec1e294d08d06 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/cf592e4784314b02a56ec1e294d08d06 2024-11-20T10:27:09,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/d5eaeae1c1684968b914d41d3e750483 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/d5eaeae1c1684968b914d41d3e750483 2024-11-20T10:27:09,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/0ff0fbd6d5fa413ebe8cce070b1864c0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/0ff0fbd6d5fa413ebe8cce070b1864c0 2024-11-20T10:27:09,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/fb82b7de834d49c78379eb08f58bcd24 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/fb82b7de834d49c78379eb08f58bcd24 2024-11-20T10:27:09,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/632862b8d4f645be9fda522e22184f4c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/deec465d31844c3cbc3c1426c8570216, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/056c2dbcdb6a4253960feccd74cea015, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/344870965bae4115a56439c71f8f772d, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/60d5c6a59aaa4c69ab51638fa37067eb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/e9cbbd86c4bc422685c9399a375b00ad, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/28e54bc43adf4418b548ec85fa827379, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/4b941a2fa7e14d5b8f982ca9176897f0, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/06dc1be11ece4ab4878b09ef3b8824a5, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/efe78a035ddc4574974ad04304a4082e, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/3ecffd10972c445a88733dd48a174755, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/fb5359f5e7db4b0094029ea807d12fbb, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/20e8528122744543a5628d6a896d4975, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/d0f85253627d452abd0a96fc3d00b9a7, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/53dbea44b9af4e308d1bd97df5191ecf, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/7f35a57cd42a4d6faf21e729aa397765, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/542e8f4714b146898d399a874da7be0c, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/c69544cae6684116a21371bf54709501] to archive 2024-11-20T10:27:09,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:27:09,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/632862b8d4f645be9fda522e22184f4c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/632862b8d4f645be9fda522e22184f4c 2024-11-20T10:27:09,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/deec465d31844c3cbc3c1426c8570216 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/deec465d31844c3cbc3c1426c8570216 2024-11-20T10:27:09,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/056c2dbcdb6a4253960feccd74cea015 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/056c2dbcdb6a4253960feccd74cea015 2024-11-20T10:27:09,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/344870965bae4115a56439c71f8f772d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/344870965bae4115a56439c71f8f772d 2024-11-20T10:27:09,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/60d5c6a59aaa4c69ab51638fa37067eb to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/60d5c6a59aaa4c69ab51638fa37067eb 2024-11-20T10:27:09,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/e9cbbd86c4bc422685c9399a375b00ad to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/e9cbbd86c4bc422685c9399a375b00ad 2024-11-20T10:27:09,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/28e54bc43adf4418b548ec85fa827379 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/28e54bc43adf4418b548ec85fa827379 2024-11-20T10:27:09,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/4b941a2fa7e14d5b8f982ca9176897f0 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/4b941a2fa7e14d5b8f982ca9176897f0 2024-11-20T10:27:09,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/06dc1be11ece4ab4878b09ef3b8824a5 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/06dc1be11ece4ab4878b09ef3b8824a5 2024-11-20T10:27:09,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/efe78a035ddc4574974ad04304a4082e to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/efe78a035ddc4574974ad04304a4082e 2024-11-20T10:27:09,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/3ecffd10972c445a88733dd48a174755 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/3ecffd10972c445a88733dd48a174755 2024-11-20T10:27:09,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/fb5359f5e7db4b0094029ea807d12fbb to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/fb5359f5e7db4b0094029ea807d12fbb 2024-11-20T10:27:09,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/20e8528122744543a5628d6a896d4975 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/20e8528122744543a5628d6a896d4975 2024-11-20T10:27:09,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/d0f85253627d452abd0a96fc3d00b9a7 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/d0f85253627d452abd0a96fc3d00b9a7 2024-11-20T10:27:09,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/53dbea44b9af4e308d1bd97df5191ecf to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/53dbea44b9af4e308d1bd97df5191ecf 2024-11-20T10:27:09,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/7f35a57cd42a4d6faf21e729aa397765 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/7f35a57cd42a4d6faf21e729aa397765 2024-11-20T10:27:09,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/542e8f4714b146898d399a874da7be0c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/542e8f4714b146898d399a874da7be0c 2024-11-20T10:27:09,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/c69544cae6684116a21371bf54709501 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/c69544cae6684116a21371bf54709501 2024-11-20T10:27:09,994 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/recovered.edits/301.seqid, newMaxSeqId=301, maxSeqId=4 2024-11-20T10:27:09,994 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. 2024-11-20T10:27:09,995 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for b746a4c7c9036a37fd4050571abf4f3f: Waiting for close lock at 1732098428229Running coprocessor pre-close hooks at 1732098428229Disabling compacts and flushes for region at 1732098428229Disabling writes for close at 1732098428230 (+1 ms)Obtaining lock to block concurrent updates at 1732098428230Preparing flush snapshotting stores in b746a4c7c9036a37fd4050571abf4f3f at 1732098428230Finished memstore snapshotting TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732098428230Flushing stores of TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f. at 1732098428231 (+1 ms)Flushing b746a4c7c9036a37fd4050571abf4f3f/A: creating writer at 1732098428231Flushing mob file b746a4c7c9036a37fd4050571abf4f3f/A: appending metadata at 1732098428236 (+5 ms)Flushing mob file b746a4c7c9036a37fd4050571abf4f3f/A: closing flushed file at 1732098428236Flushing b746a4c7c9036a37fd4050571abf4f3f/A: appending metadata at 1732098428654 (+418 ms)Flushing b746a4c7c9036a37fd4050571abf4f3f/A: closing flushed file at 1732098428654Flushing b746a4c7c9036a37fd4050571abf4f3f/B: creating writer at 1732098429065 (+411 ms)Flushing b746a4c7c9036a37fd4050571abf4f3f/B: appending metadata at 1732098429067 (+2 ms)Flushing b746a4c7c9036a37fd4050571abf4f3f/B: closing flushed file at 1732098429067Flushing b746a4c7c9036a37fd4050571abf4f3f/C: creating writer at 1732098429481 (+414 ms)Flushing b746a4c7c9036a37fd4050571abf4f3f/C: appending metadata at 1732098429484 (+3 ms)Flushing b746a4c7c9036a37fd4050571abf4f3f/C: closing flushed file at 1732098429484Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1aab286b: reopening flushed file at 1732098429900 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f8bf8f0: reopening flushed file at 1732098429905 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55e54132: reopening flushed file at 1732098429909 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for b746a4c7c9036a37fd4050571abf4f3f in 1684ms, sequenceid=298, compaction requested=true at 1732098429914 (+5 ms)Writing region close event to WAL at 1732098429991 (+77 ms)Running coprocessor post-close hooks at 1732098429994 (+3 ms)Closed at 1732098429994 2024-11-20T10:27:09,996 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:09,997 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=b746a4c7c9036a37fd4050571abf4f3f, regionState=CLOSED 2024-11-20T10:27:09,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 because future has completed 2024-11-20T10:27:09,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-20T10:27:10,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure b746a4c7c9036a37fd4050571abf4f3f, server=8c533afd331c,35833,1732098238678 in 1.9200 sec 2024-11-20T10:27:10,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-20T10:27:10,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b746a4c7c9036a37fd4050571abf4f3f, UNASSIGN in 1.9260 sec 2024-11-20T10:27:10,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-20T10:27:10,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9290 sec 2024-11-20T10:27:10,003 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098430003"}]},"ts":"1732098430003"} 2024-11-20T10:27:10,004 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T10:27:10,004 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T10:27:10,005 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9420 sec 2024-11-20T10:27:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T10:27:10,218 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:27:10,219 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T10:27:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:27:10,224 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:27:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T10:27:10,226 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:27:10,230 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,234 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C, FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/recovered.edits] 2024-11-20T10:27:10,239 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/10a4a9545d90423da8da4a1126e18886 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/10a4a9545d90423da8da4a1126e18886 2024-11-20T10:27:10,241 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/a117167e4ba3419ca0d7c14b8678ee13 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/a117167e4ba3419ca0d7c14b8678ee13 2024-11-20T10:27:10,243 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/f392705e2c4b4482829939d69c402062 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/A/f392705e2c4b4482829939d69c402062 2024-11-20T10:27:10,247 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/aba5117eb05f47c097d7629aa215049c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/aba5117eb05f47c097d7629aa215049c 2024-11-20T10:27:10,249 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/d0deadcdc546472b8a04ea03309cc6d1 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/d0deadcdc546472b8a04ea03309cc6d1 2024-11-20T10:27:10,251 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/dea53077c2db4bbbac4c74f2f973ea65 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/B/dea53077c2db4bbbac4c74f2f973ea65 2024-11-20T10:27:10,255 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/1b649bdfa1e6450d9751fbe63dd8815d to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/1b649bdfa1e6450d9751fbe63dd8815d 2024-11-20T10:27:10,257 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/402ccb65c1b1474bbc6557d586be9db9 to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/402ccb65c1b1474bbc6557d586be9db9 2024-11-20T10:27:10,259 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/bfa084981f2240379a8e612554ec403c to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/C/bfa084981f2240379a8e612554ec403c 2024-11-20T10:27:10,263 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/recovered.edits/301.seqid to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f/recovered.edits/301.seqid 2024-11-20T10:27:10,264 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/default/TestAcidGuarantees/b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,264 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T10:27:10,265 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T10:27:10,266 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T10:27:10,271 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120035901fd36f5489ba7080c93e6ec9c45_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120035901fd36f5489ba7080c93e6ec9c45_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,274 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120171a5540c328461a926bee1aa14f2830_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120171a5540c328461a926bee1aa14f2830_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,276 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120214f1369698b48b2802623c2ee5cb034_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120214f1369698b48b2802623c2ee5cb034_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,278 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120245c90beddbe42499fc1edba37852423_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120245c90beddbe42499fc1edba37852423_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,280 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120280ac3587da4435e822f4ccb8c2632e4_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120280ac3587da4435e822f4ccb8c2632e4_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,282 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202d314e4d057247399b945100e5167a53_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202d314e4d057247399b945100e5167a53_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,283 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203f18894113d544628a6de49dd4fa5bcc_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203f18894113d544628a6de49dd4fa5bcc_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,285 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204259cb24d12e42289f33a16725cf8f3a_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204259cb24d12e42289f33a16725cf8f3a_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,287 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112051d0179bc2984e1e91eaae16aa3c6e73_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112051d0179bc2984e1e91eaae16aa3c6e73_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,288 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120592ad8a2ca0b498d80b6496796cee53a_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120592ad8a2ca0b498d80b6496796cee53a_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,290 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205e90ec590c984f90812f5fe38b4aec0a_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205e90ec590c984f90812f5fe38b4aec0a_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,291 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206d773b0e0c8247b6ae0e2ac05f66d96f_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206d773b0e0c8247b6ae0e2ac05f66d96f_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,293 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207b5fb9e2e2134bf885edd91493564d82_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207b5fb9e2e2134bf885edd91493564d82_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,295 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207f0a2da89c6844a9a9b19eefb5f38538_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207f0a2da89c6844a9a9b19eefb5f38538_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,296 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b06054c007d44866b2a37f62aff4a154_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b06054c007d44866b2a37f62aff4a154_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,298 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c77a4c2674734935a279bd99c56aab70_b746a4c7c9036a37fd4050571abf4f3f to hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c77a4c2674734935a279bd99c56aab70_b746a4c7c9036a37fd4050571abf4f3f 2024-11-20T10:27:10,299 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T10:27:10,302 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:27:10,306 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T10:27:10,309 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T10:27:10,310 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:27:10,310 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T10:27:10,310 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732098430310"}]},"ts":"9223372036854775807"} 2024-11-20T10:27:10,313 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T10:27:10,313 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b746a4c7c9036a37fd4050571abf4f3f, NAME => 'TestAcidGuarantees,,1732098403682.b746a4c7c9036a37fd4050571abf4f3f.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T10:27:10,313 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T10:27:10,313 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732098430313"}]},"ts":"9223372036854775807"} 2024-11-20T10:27:10,316 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T10:27:10,317 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T10:27:10,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 98 msec 2024-11-20T10:27:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33867 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T10:27:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T10:27:10,338 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T10:27:10,350 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242 (was 236) - Thread LEAK? -, OpenFileDescriptor=457 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=407 (was 450), ProcessCount=11 (was 11), AvailableMemoryMB=8084 (was 8110) 2024-11-20T10:27:10,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T10:27:10,350 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T10:27:10,350 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T10:27:10,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:10,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:10,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:27:10,350 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T10:27:10,351 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=694036216, stopped=false 2024-11-20T10:27:10,351 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8c533afd331c,33867,1732098237476 2024-11-20T10:27:10,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T10:27:10,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T10:27:10,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:27:10,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:27:10,397 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T10:27:10,398 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T10:27:10,398 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T10:27:10,398 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T10:27:10,399 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T10:27:10,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:10,400 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8c533afd331c,35833,1732098238678' ***** 2024-11-20T10:27:10,400 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T10:27:10,400 INFO [RS:0;8c533afd331c:35833 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T10:27:10,401 INFO [RS:0;8c533afd331c:35833 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T10:27:10,401 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T10:27:10,401 INFO [RS:0;8c533afd331c:35833 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T10:27:10,401 INFO [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(959): stopping server 8c533afd331c,35833,1732098238678 2024-11-20T10:27:10,401 INFO [RS:0;8c533afd331c:35833 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T10:27:10,402 INFO [RS:0;8c533afd331c:35833 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8c533afd331c:35833. 2024-11-20T10:27:10,402 DEBUG [RS:0;8c533afd331c:35833 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T10:27:10,402 DEBUG [RS:0;8c533afd331c:35833 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:10,402 INFO [RS:0;8c533afd331c:35833 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T10:27:10,402 INFO [RS:0;8c533afd331c:35833 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T10:27:10,402 INFO [RS:0;8c533afd331c:35833 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T10:27:10,402 INFO [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T10:27:10,403 INFO [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T10:27:10,404 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-20T10:27:10,404 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T10:27:10,404 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T10:27:10,404 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T10:27:10,404 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T10:27:10,405 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T10:27:10,405 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T10:27:10,405 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-20T10:27:10,424 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/.tmp/info/740d2cfefd6548769c7db794f3426ecb is 95, key is TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4./info:/1732098278428/DeleteFamily/seqid=0 2024-11-20T10:27:10,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742448_1624 (size=6514) 2024-11-20T10:27:10,450 INFO [regionserver/8c533afd331c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T10:27:10,605 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T10:27:10,805 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T10:27:10,828 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/.tmp/info/740d2cfefd6548769c7db794f3426ecb 2024-11-20T10:27:10,851 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/.tmp/ns/1ad8dab246464b78a1277d5329e1ba25 is 93, key is TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4./ns:/1732098278382/DeleteFamily/seqid=0 2024-11-20T10:27:10,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742449_1625 (size=5906) 2024-11-20T10:27:10,865 INFO [regionserver/8c533afd331c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T10:27:10,865 INFO [regionserver/8c533afd331c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T10:27:11,005 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T10:27:11,205 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T10:27:11,256 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/.tmp/ns/1ad8dab246464b78a1277d5329e1ba25 2024-11-20T10:27:11,281 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/.tmp/rep_barrier/4d8416b9471541d9b8be8312f9e6dea6 is 102, key is TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4./rep_barrier:/1732098278382/DeleteFamily/seqid=0 2024-11-20T10:27:11,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742450_1626 (size=6025) 2024-11-20T10:27:11,406 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T10:27:11,606 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T10:27:11,686 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/.tmp/rep_barrier/4d8416b9471541d9b8be8312f9e6dea6 2024-11-20T10:27:11,710 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/.tmp/table/54d8a23a5e7c4f72a1a11040f9557985 is 96, key is TestAcidGuarantees,,1732098243494.455386d186b9f64def53c239491339e4./table:/1732098278382/DeleteFamily/seqid=0 2024-11-20T10:27:11,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742451_1627 (size=5936) 2024-11-20T10:27:11,806 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T10:27:12,007 DEBUG [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T10:27:12,115 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/.tmp/table/54d8a23a5e7c4f72a1a11040f9557985 2024-11-20T10:27:12,125 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/.tmp/info/740d2cfefd6548769c7db794f3426ecb as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/info/740d2cfefd6548769c7db794f3426ecb 2024-11-20T10:27:12,129 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/info/740d2cfefd6548769c7db794f3426ecb, entries=12, sequenceid=90, filesize=6.4 K 2024-11-20T10:27:12,130 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/.tmp/ns/1ad8dab246464b78a1277d5329e1ba25 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/ns/1ad8dab246464b78a1277d5329e1ba25 2024-11-20T10:27:12,135 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/ns/1ad8dab246464b78a1277d5329e1ba25, entries=8, sequenceid=90, filesize=5.8 K 2024-11-20T10:27:12,135 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/.tmp/rep_barrier/4d8416b9471541d9b8be8312f9e6dea6 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/rep_barrier/4d8416b9471541d9b8be8312f9e6dea6 2024-11-20T10:27:12,139 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/rep_barrier/4d8416b9471541d9b8be8312f9e6dea6, entries=6, sequenceid=90, filesize=5.9 K 2024-11-20T10:27:12,139 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/.tmp/table/54d8a23a5e7c4f72a1a11040f9557985 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/table/54d8a23a5e7c4f72a1a11040f9557985 2024-11-20T10:27:12,143 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/table/54d8a23a5e7c4f72a1a11040f9557985, entries=7, sequenceid=90, filesize=5.8 K 2024-11-20T10:27:12,143 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1738ms, sequenceid=90, compaction requested=false 2024-11-20T10:27:12,148 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-20T10:27:12,149 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T10:27:12,149 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T10:27:12,149 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732098430404Running coprocessor pre-close hooks at 1732098430404Disabling compacts and flushes for region at 1732098430404Disabling writes for close at 1732098430405 (+1 ms)Obtaining lock to block concurrent updates at 1732098430405Preparing flush snapshotting stores in 1588230740 at 1732098430405Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732098430405Flushing stores of hbase:meta,,1.1588230740 at 1732098430406 (+1 ms)Flushing 1588230740/info: creating writer at 1732098430406Flushing 1588230740/info: appending metadata at 1732098430424 (+18 ms)Flushing 1588230740/info: closing flushed file at 1732098430424Flushing 1588230740/ns: creating writer at 1732098430836 (+412 ms)Flushing 1588230740/ns: appending metadata at 1732098430851 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732098430851Flushing 1588230740/rep_barrier: creating writer at 1732098431265 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732098431281 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732098431281Flushing 1588230740/table: creating writer at 1732098431694 (+413 ms)Flushing 1588230740/table: appending metadata at 1732098431710 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732098431710Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@549ca2e: reopening flushed file at 1732098432124 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cf00d3c: reopening flushed file at 1732098432130 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b035b30: reopening flushed file at 1732098432135 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ba2ffc0: reopening flushed file at 1732098432139 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1738ms, sequenceid=90, compaction requested=false at 1732098432143 (+4 ms)Writing region close event to WAL at 1732098432145 (+2 ms)Running coprocessor post-close hooks at 1732098432149 (+4 ms)Closed at 1732098432149 2024-11-20T10:27:12,150 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T10:27:12,207 INFO [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(976): stopping server 8c533afd331c,35833,1732098238678; all regions closed. 2024-11-20T10:27:12,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741834_1010 (size=25344) 2024-11-20T10:27:12,220 DEBUG [RS:0;8c533afd331c:35833 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/oldWALs 2024-11-20T10:27:12,220 INFO [RS:0;8c533afd331c:35833 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8c533afd331c%2C35833%2C1732098238678.meta:.meta(num 1732098243027) 2024-11-20T10:27:12,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741832_1008 (size=14154588) 2024-11-20T10:27:12,226 DEBUG [RS:0;8c533afd331c:35833 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/oldWALs 2024-11-20T10:27:12,226 INFO [RS:0;8c533afd331c:35833 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8c533afd331c%2C35833%2C1732098238678:(num 1732098241991) 2024-11-20T10:27:12,226 DEBUG [RS:0;8c533afd331c:35833 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:27:12,226 INFO [RS:0;8c533afd331c:35833 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T10:27:12,226 INFO [RS:0;8c533afd331c:35833 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T10:27:12,226 INFO [RS:0;8c533afd331c:35833 {}] hbase.ChoreService(370): Chore service for: regionserver/8c533afd331c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T10:27:12,227 INFO [RS:0;8c533afd331c:35833 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T10:27:12,227 INFO [regionserver/8c533afd331c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T10:27:12,227 INFO [RS:0;8c533afd331c:35833 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35833 2024-11-20T10:27:12,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8c533afd331c,35833,1732098238678 2024-11-20T10:27:12,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T10:27:12,285 INFO [RS:0;8c533afd331c:35833 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T10:27:12,295 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8c533afd331c,35833,1732098238678] 2024-11-20T10:27:12,305 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8c533afd331c,35833,1732098238678 already deleted, retry=false 2024-11-20T10:27:12,306 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8c533afd331c,35833,1732098238678 expired; onlineServers=0 2024-11-20T10:27:12,306 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8c533afd331c,33867,1732098237476' ***** 2024-11-20T10:27:12,306 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T10:27:12,306 INFO [M:0;8c533afd331c:33867 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T10:27:12,307 INFO [M:0;8c533afd331c:33867 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T10:27:12,307 DEBUG [M:0;8c533afd331c:33867 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T10:27:12,307 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T10:27:12,307 DEBUG [M:0;8c533afd331c:33867 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T10:27:12,308 DEBUG [master/8c533afd331c:0:becomeActiveMaster-HFileCleaner.small.0-1732098241688 {}] cleaner.HFileCleaner(306): Exit Thread[master/8c533afd331c:0:becomeActiveMaster-HFileCleaner.small.0-1732098241688,5,FailOnTimeoutGroup] 2024-11-20T10:27:12,308 DEBUG [master/8c533afd331c:0:becomeActiveMaster-HFileCleaner.large.0-1732098241686 {}] cleaner.HFileCleaner(306): Exit Thread[master/8c533afd331c:0:becomeActiveMaster-HFileCleaner.large.0-1732098241686,5,FailOnTimeoutGroup] 2024-11-20T10:27:12,308 INFO [M:0;8c533afd331c:33867 {}] hbase.ChoreService(370): Chore service for: master/8c533afd331c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T10:27:12,309 INFO [M:0;8c533afd331c:33867 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T10:27:12,309 DEBUG [M:0;8c533afd331c:33867 {}] master.HMaster(1795): Stopping service threads 2024-11-20T10:27:12,309 INFO [M:0;8c533afd331c:33867 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T10:27:12,309 INFO [M:0;8c533afd331c:33867 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T10:27:12,310 ERROR [M:0;8c533afd331c:33867 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (205990495) connection to localhost/127.0.0.1:45087 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:45087,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-20T10:27:12,311 INFO [M:0;8c533afd331c:33867 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T10:27:12,312 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T10:27:12,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T10:27:12,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:27:12,316 DEBUG [M:0;8c533afd331c:33867 {}] zookeeper.ZKUtil(347): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T10:27:12,316 WARN [M:0;8c533afd331c:33867 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T10:27:12,316 INFO [M:0;8c533afd331c:33867 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/.lastflushedseqids 2024-11-20T10:27:12,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742452_1628 (size=72) 2024-11-20T10:27:12,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T10:27:12,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10157c59c580001, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T10:27:12,397 INFO [RS:0;8c533afd331c:35833 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T10:27:12,397 INFO [RS:0;8c533afd331c:35833 {}] regionserver.HRegionServer(1031): Exiting; stopping=8c533afd331c,35833,1732098238678; zookeeper connection closed. 2024-11-20T10:27:12,398 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@66e67bad {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@66e67bad 2024-11-20T10:27:12,399 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-20T10:27:12,727 INFO [M:0;8c533afd331c:33867 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T10:27:12,727 INFO [M:0;8c533afd331c:33867 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T10:27:12,728 DEBUG [M:0;8c533afd331c:33867 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T10:27:12,728 INFO [M:0;8c533afd331c:33867 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:27:12,728 DEBUG [M:0;8c533afd331c:33867 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:27:12,728 DEBUG [M:0;8c533afd331c:33867 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T10:27:12,728 DEBUG [M:0;8c533afd331c:33867 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:27:12,728 INFO [M:0;8c533afd331c:33867 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=750.08 KB heapSize=921.43 KB 2024-11-20T10:27:12,753 DEBUG [M:0;8c533afd331c:33867 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f2f08ad5111f4e0bad7936d7cbb00ea5 is 82, key is hbase:meta,,1/info:regioninfo/1732098243112/Put/seqid=0 2024-11-20T10:27:12,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742453_1629 (size=5672) 2024-11-20T10:27:13,158 INFO [M:0;8c533afd331c:33867 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2107 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f2f08ad5111f4e0bad7936d7cbb00ea5 2024-11-20T10:27:13,186 DEBUG [M:0;8c533afd331c:33867 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b7c8b76b102c46b6bd2a43b72860c62a is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x97/proc:d/1732098406744/Put/seqid=0 2024-11-20T10:27:13,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742454_1630 (size=45171) 2024-11-20T10:27:13,591 INFO [M:0;8c533afd331c:33867 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=749.52 KB at sequenceid=2107 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b7c8b76b102c46b6bd2a43b72860c62a 2024-11-20T10:27:13,601 INFO [M:0;8c533afd331c:33867 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b7c8b76b102c46b6bd2a43b72860c62a 2024-11-20T10:27:13,617 DEBUG [M:0;8c533afd331c:33867 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/33f5763daf6b4194a0023d6a41ecc078 is 69, key is 8c533afd331c,35833,1732098238678/rs:state/1732098241731/Put/seqid=0 2024-11-20T10:27:13,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073742455_1631 (size=5156) 2024-11-20T10:27:14,021 INFO [M:0;8c533afd331c:33867 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2107 (bloomFilter=true), to=hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/33f5763daf6b4194a0023d6a41ecc078 2024-11-20T10:27:14,027 DEBUG [M:0;8c533afd331c:33867 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f2f08ad5111f4e0bad7936d7cbb00ea5 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f2f08ad5111f4e0bad7936d7cbb00ea5 2024-11-20T10:27:14,031 INFO [M:0;8c533afd331c:33867 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f2f08ad5111f4e0bad7936d7cbb00ea5, entries=8, sequenceid=2107, filesize=5.5 K 2024-11-20T10:27:14,033 DEBUG [M:0;8c533afd331c:33867 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b7c8b76b102c46b6bd2a43b72860c62a as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b7c8b76b102c46b6bd2a43b72860c62a 2024-11-20T10:27:14,037 INFO [M:0;8c533afd331c:33867 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b7c8b76b102c46b6bd2a43b72860c62a 2024-11-20T10:27:14,037 INFO [M:0;8c533afd331c:33867 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b7c8b76b102c46b6bd2a43b72860c62a, entries=176, sequenceid=2107, filesize=44.1 K 2024-11-20T10:27:14,038 DEBUG [M:0;8c533afd331c:33867 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/33f5763daf6b4194a0023d6a41ecc078 as hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/33f5763daf6b4194a0023d6a41ecc078 2024-11-20T10:27:14,042 INFO [M:0;8c533afd331c:33867 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45087/user/jenkins/test-data/29825b52-df1a-ea25-2cab-ffde798cde31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/33f5763daf6b4194a0023d6a41ecc078, entries=1, sequenceid=2107, filesize=5.0 K 2024-11-20T10:27:14,043 INFO [M:0;8c533afd331c:33867 {}] regionserver.HRegion(3140): Finished flush of dataSize ~750.08 KB/768077, heapSize ~921.13 KB/943240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1315ms, sequenceid=2107, compaction requested=false 2024-11-20T10:27:14,044 INFO [M:0;8c533afd331c:33867 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:27:14,045 DEBUG [M:0;8c533afd331c:33867 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732098432728Disabling compacts and flushes for region at 1732098432728Disabling writes for close at 1732098432728Obtaining lock to block concurrent updates at 1732098432728Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732098432728Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=768077, getHeapSize=943480, getOffHeapSize=0, getCellsCount=2308 at 1732098432729 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732098432731 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732098432731Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732098432753 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732098432753Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732098433168 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732098433186 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732098433186Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732098433601 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732098433616 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732098433616Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a40cb42: reopening flushed file at 1732098434026 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dbffcb4: reopening flushed file at 1732098434032 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72c64731: reopening flushed file at 1732098434037 (+5 ms)Finished flush of dataSize ~750.08 KB/768077, heapSize ~921.13 KB/943240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1315ms, sequenceid=2107, compaction requested=false at 1732098434043 (+6 ms)Writing region close event to WAL at 1732098434044 (+1 ms)Closed at 1732098434044 2024-11-20T10:27:14,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37423 is added to blk_1073741830_1006 (size=906255) 2024-11-20T10:27:14,048 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T10:27:14,048 INFO [M:0;8c533afd331c:33867 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T10:27:14,048 INFO [M:0;8c533afd331c:33867 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33867 2024-11-20T10:27:14,048 INFO [M:0;8c533afd331c:33867 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T10:27:14,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T10:27:14,216 INFO [M:0;8c533afd331c:33867 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T10:27:14,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33867-0x10157c59c580000, quorum=127.0.0.1:55353, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T10:27:14,225 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@51084f0f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T10:27:14,229 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@370621d9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T10:27:14,229 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T10:27:14,229 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4828e07b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T10:27:14,230 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6ff64ba0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/hadoop.log.dir/,STOPPED} 2024-11-20T10:27:14,233 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T10:27:14,233 WARN [BP-606364108-172.17.0.2-1732098233200 heartbeating to localhost/127.0.0.1:45087 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T10:27:14,233 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T10:27:14,233 WARN [BP-606364108-172.17.0.2-1732098233200 heartbeating to localhost/127.0.0.1:45087 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-606364108-172.17.0.2-1732098233200 (Datanode Uuid ae59ac02-3972-44ab-8c09-49370f1bb9eb) service to localhost/127.0.0.1:45087 2024-11-20T10:27:14,237 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/cluster_df478fc5-be47-d3ac-f609-45ae39353117/data/data1/current/BP-606364108-172.17.0.2-1732098233200 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T10:27:14,237 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/cluster_df478fc5-be47-d3ac-f609-45ae39353117/data/data2/current/BP-606364108-172.17.0.2-1732098233200 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T10:27:14,237 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T10:27:14,243 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T10:27:14,244 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T10:27:14,244 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T10:27:14,244 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T10:27:14,244 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b447d74b-c436-d597-7208-d4653059b605/hadoop.log.dir/,STOPPED} 2024-11-20T10:27:14,257 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T10:27:14,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down